diff --git a/.secrets.baseline b/.secrets.baseline index 57fa3236f27..b815a28edc9 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -1015,6 +1015,15 @@ "line_number": 1108 } ], + "infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml": [ + { + "type": "Secret Keyword", + "filename": "infra/feast-operator/bundle/manifests/openlineage-secret_v1_secret.yaml", + "hashed_secret": "598319ac6aa4a94e72a9d8a8d405cc7bc9e048ee", + "is_verified": false, + "line_number": 6 + } + ], "infra/feast-operator/config/samples/v1_featurestore_db_persistence.yaml": [ { "type": "Secret Keyword", diff --git a/README.md b/README.md index 0ea5b0a135a..a91faccae83 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ ## Join us on Slack! -👋👋👋 [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +👋👋👋 [Come say hi on Slack!](https://slack.feast.dev/) [Check out our DeepWiki!](https://deepwiki.com/feast-dev/feast) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index d35dbf1652b..f8d04186743 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -205,7 +205,10 @@ * [\[Beta\] On demand feature view](reference/beta-on-demand-feature-view.md) * [\[Alpha\] Static Artifacts Loading](reference/alpha-static-artifacts.md) * [\[Alpha\] Vector Database](reference/alpha-vector-database.md) +* [\[Alpha\] OpenAI-Compatible Vector Store API](reference/alpha-vector-database.md#alpha-openai-compatible-vector-store-api) + * [Data Quality Monitoring](reference/dqm.md) +* [\[Deprecated\] Data quality monitoring (Great Expectations)](reference/dqm.md) * [\[Alpha\] Streaming feature computation with Denormalized](reference/denormalized.md) * [\[Alpha\] Feature View Versioning](reference/alpha-feature-view-versioning.md) * [OpenLineage Integration](reference/openlineage.md) diff --git a/docs/community.md b/docs/community.md index 640b5238b8b..c22c4c41dd4 100644 --- a/docs/community.md +++ b/docs/community.md @@ -2,7 +2,7 @@ ## Links & Resources -* [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +* [Come say hi on Slack!](https://slack.feast.dev/) * As a part of the Linux Foundation, we ask community members to adhere to the [Linux Foundation Code of Conduct](https://events.linuxfoundation.org/about/code-of-conduct/) * [GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. * [Community Governance Doc](https://github.com/feast-dev/feast/blob/master/community): See the governance model of Feast, including who the maintainers are and how decisions are made. diff --git a/docs/getting-started/components/feature-server.md b/docs/getting-started/components/feature-server.md index 4d961054ecb..1b5521a2b86 100644 --- a/docs/getting-started/components/feature-server.md +++ b/docs/getting-started/components/feature-server.md @@ -37,6 +37,8 @@ The Feature Server operates as a stateless service backed by two key components: | `/push` | Pushes feature data to the online and/or offline store. | | `/materialize` | Materializes features within a specific time range to the online store. | | `/materialize-incremental` | Incrementally materializes features up to the current timestamp. | -| `/retrieve-online-documents` | Supports Vector Similarity Search for RAG (Alpha end-ponit) | +| `/search` | Vector similarity search for RAG (Alpha endpoint) | +| `/v1/vector_stores/{id}/search` | OpenAI-compatible vector search with server-side embedding | +| `/retrieve-online-documents` | **Deprecated.** Use `/search` instead. | | `/docs` | API Contract for available endpoints | diff --git a/docs/getting-started/genai.md b/docs/getting-started/genai.md index dfd9c41954a..d9f682af1f0 100644 --- a/docs/getting-started/genai.md +++ b/docs/getting-started/genai.md @@ -226,7 +226,8 @@ The MCP integration uses the `fastapi_mcp` library to automatically transform yo The fastapi_mcp integration automatically exposes your Feast feature server's FastAPI endpoints as MCP tools. This means AI assistants can: * **Call `/get-online-features`** to retrieve features from the feature store -* **Call `/retrieve-online-documents`** to perform vector similarity search +* **Call `/search`** to perform vector similarity search (`/retrieve-online-documents` is a deprecated alias) +* **Call `/v1/vector_stores/{feature_view}/search`** for OpenAI-compatible text search with server-side embedding * **Call `/write-to-online-store`** to persist agent state (memory, notes, interaction history) * **Use `/health`** to check server status diff --git a/docs/reference/alpha-vector-database.md b/docs/reference/alpha-vector-database.md index 28d0bf0098e..61da02ce6f4 100644 --- a/docs/reference/alpha-vector-database.md +++ b/docs/reference/alpha-vector-database.md @@ -33,6 +33,240 @@ backwards compatibility and the adopt industry standard naming conventions. **Note**: Milvus, SQLite, and ScyllaDB implement the v2 `retrieve_online_documents_v2` method in the SDK. This will be the longer-term solution so that Data Scientists can easily enable vector similarity search by just flipping a flag. +## Feature server search endpoints + +| Endpoint | Use when | +|----------|----------| +| `POST /search` | You have an embedding vector (or use `api_version: 2` with `query_string`) and want Feast's native online-features response format. | +| `GET /v1/vector_stores` | You want to discover available vector stores and their `vs_{hash}` IDs (OpenAI-compatible). | +| `GET /v1/vector_stores/{id}` | You want metadata for a specific vector store (OpenAI-compatible). | +| `POST /v1/vector_stores/{id}/search` | You want plain-text queries with server-side embedding and an OpenAI-compatible response. | + +`POST /retrieve-online-documents` is deprecated; use `POST /search` instead. + +## [Alpha] OpenAI-Compatible Vector Store API + +{% hint style="warning" %} +**Alpha feature.** This API surface is functional and tested, but may change in future releases. Feedback and contributions are welcome. +{% endhint %} + +Feast exposes a set of [OpenAI-compatible vector store endpoints](https://platform.openai.com/docs/api-reference/vector-stores) that let clients discover, inspect, and search vector stores using plain text queries with server-side embedding. This enables integration with AI agents, LLM tool-calling frameworks, and any OpenAI-compatible client without requiring the caller to produce raw embedding vectors. + +### Vector store IDs + +Each feature view with at least one `vector_index=True` field is automatically assigned a deterministic identifier of the form `vs_{hash}`, where `{hash}` is the first 24 characters of `SHA-256(project + ":" + feature_view_name)`. These IDs are stable across server restarts and registry refreshes. + +For example, a feature view named `product_catalog` in project `my_project` always maps to the same `vs_...` identifier. The listing endpoints return these IDs so clients can discover stores at runtime. + +### Endpoints + +| Method | Path | Permission | Description | +|--------|------|------------|-------------| +| `GET` | `/v1/vector_stores` | `DESCRIBE` | List all vector stores the caller has access to | +| `GET` | `/v1/vector_stores/{vector_store_id}` | `DESCRIBE` | Get metadata for a single vector store | +| `POST` | `/v1/vector_stores/{vector_store_id}/search` | `READ_ONLINE` | Search a vector store with a plain text query | + +All endpoints enforce RBAC when authentication is configured. The listing endpoint filters out stores the caller cannot `DESCRIBE`. + +### Requirements + +1. **Embedding model** — an `embedding_model` section in `feature_store.yaml`. Feast uses [Sentence Transformers](https://www.sbert.net/) by default for local embedding — no external API key required (`pip install sentence-transformers`): + + ```yaml + embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 + ``` + +2. **Vector-indexed feature view** — at least one feature view with `vector_index=True` on a vector field, materialized to an online store that supports vector search. + +3. **Numeric filtering (optional)** — for metadata filters that use numeric or boolean comparisons, set `enable_openai_compatible_store: true` on your online store config and run `feast apply` to add the required `value_num` column. + +### Custom embedding providers + +The built-in Sentence Transformers provider works for most use cases. To use a different embedding backend (OpenAI, Cohere, a custom model, etc.), implement the `EmbeddingProvider` protocol and pass an instance to `FeatureStore`: + +```python +from feast.embedder import EmbeddingProvider + +class MyEmbeddingProvider: + def embed(self, texts: list[str]) -> list[list[float]]: + # Call your embedding API here + return my_model.encode(texts) + + async def aembed(self, texts: list[str]) -> list[list[float]]: + return await my_model.aencode(texts) + +store = FeatureStore( + repo_path=".", + embedding_provider=MyEmbeddingProvider(), +) +``` + +### Numeric storage (`enable_openai_compatible_store`) + +By default, feature values are stored as text in the online store. This means string-ordered comparisons apply (e.g., `'9' > '100'` is `true`). When `enable_openai_compatible_store: true` is set on the online store config, Feast adds a `value_num` column that stores `int`, `float`, `double`, and `bool` values natively so that numeric filters produce correct results. + +```yaml +online_store: + type: postgres # or sqlite + # ... connection settings ... + enable_openai_compatible_store: true +``` + +After changing this setting, run `feast apply` to update the database schema. + +### List vector stores + +```bash +curl http://localhost:6566/v1/vector_stores +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "object": "vector_store", + "name": "product_catalog", + "status": "completed", + "created_at": 1717200000 + } + ] +} +``` + +### Get a single vector store + +```bash +curl http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6 +``` + +Returns the same object shape as a single entry in the list response. Returns `404` if the ID does not match any vector-indexed feature view. + +### Search + +Start the feature server with `feast serve`, then send a search request: + +```bash +curl -X POST http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6/search \ + -H "Content-Type: application/json" \ + -d '{ + "query": "wireless noise-cancelling headphones", + "max_num_results": 5 + }' +``` + +#### Request fields + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `query` | `string` or `list[string]` | (required) | Plain text search query. Lists are joined with spaces before embedding. | +| `max_num_results` | `int` | `10` | Maximum number of results to return. | +| `filters` | `object` | `null` | OpenAI-style filters (see below). | +| `ranking_options` | `object` | `null` | Accepted for forward compatibility, but currently ignored. Setting `score_threshold` or `ranker` inside it will return a 422 error. | +| `rewrite_query` | `bool` | `null` | `false` (the default/no-op) is accepted. `true` is not yet supported and will return a 422 error. | +| `metadata` | `object` | `null` | Optional. `metadata.features_to_retrieve` selects specific features. | + +### Filters + +The endpoint supports OpenAI-style filters for narrowing results beyond vector similarity. + +**Comparison operators:** `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin` + +```json +{"type": "eq", "key": "category", "value": "Electronics"} +``` + +**Compound operators:** `and`, `or` (nest to arbitrary depth) + +```json +{ + "type": "and", + "filters": [ + {"type": "eq", "key": "category", "value": "Electronics"}, + {"type": "gte", "key": "rating", "value": 4.5} + ] +} +``` + +For Postgres and SQLite backends, all filtering (including string equality) requires `enable_openai_compatible_store: true` in the online store config. After enabling, run `feast apply` to update the database schema. + +ScyllaDB supports vector retrieval via `retrieve_online_documents_v2`, but OpenAI-style metadata filtering is not implemented yet. Passing `filters` raises `NotImplementedError`. + +### Response format + +Responses follow the OpenAI `vector_store.search_results.page` schema: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": ["wireless noise-cancelling headphones"], + "data": [ + { + "file_id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6_42", + "filename": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "score": 0.92, + "attributes": {"name": "...", "category": "..."}, + "content": [ + {"type": "text", "text": "..."} + ] + } + ], + "has_more": false, + "next_page": null +} +``` + +The `file_id` and `filename` fields use the `vs_{hash}` identifier, not raw feature view names. + +The `score` field is a higher-is-better relevance score derived from the raw vector distance using a metric-dependent conversion: + +| Distance metric | Conversion | Range | +|----------------|------------|-------| +| L2 (default) | `1 / (1 + distance)` | (0, 1] | +| Cosine | `1 - distance` | [0, 1] | +| Inner product / dot | `-distance` | varies | + +The metric is determined by `vector_search_metric` on the feature view's vector field, not by an API parameter. When `features_to_retrieve` is omitted, all non-vector features are returned by default (vector embedding columns are excluded). + +Pagination is not yet implemented; `has_more` is always `false`. + +### SDK usage + +The OpenAI-compatible search is also available directly via the Python SDK: + +```python +import asyncio +from feast import FeatureStore + +store = FeatureStore(repo_path=".") + +result = asyncio.run(store.openai_search( + vector_store_id="product_catalog", + query="wireless noise-cancelling headphones", + max_num_results=5, + filters={"type": "eq", "key": "category", "value": "Electronics"}, +)) + +for item in result["data"]: + print(f"{item['score']:.3f} {item['attributes']}") +``` + +### Supported online stores + +The OpenAI-compatible filtering has been implemented for the following online stores: + +| Online Store | Vector Search | Metadata Filtering | Notes | +|-------------|--------------|-------------------|-------| +| Milvus | Yes | Yes | Boolean expressions | +| Elasticsearch | Yes | Yes | Query DSL clauses | +| Postgres (pgvector) | Yes | Yes | Requires `enable_openai_compatible_store: true` | +| SQLite (sqlite-vec) | Yes | Yes | Requires `enable_openai_compatible_store: true` | +| MongoDB | Yes | Yes | Aggregation pipeline | +| ScyllaDB | Yes | No | Vector search only; metadata filters are not supported yet | + ## Examples - See the v0 [Rag Demo](https://github.com/feast-dev/feast-workshop/blob/rag/module_4_rag) for an example on how to use vector database using the `retrieve_online_documents` method (planning migration and deprecation (planning migration and deprecation). diff --git a/docs/reference/feature-servers/python-feature-server.md b/docs/reference/feature-servers/python-feature-server.md index 4802599866d..b1b873cc7d2 100644 --- a/docs/reference/feature-servers/python-feature-server.md +++ b/docs/reference/feature-servers/python-feature-server.md @@ -527,6 +527,42 @@ Prometheus adds an `instance` label per pod, so there is no duplication. Use `sum(rate(...))` or `histogram_quantile(...)` across instances as usual. +## Vector Search (`POST /search`) + +The feature server exposes `POST /search` for vector similarity search against online document embeddings. Pass a pre-computed embedding in `query`, or use `api_version: 2` with `query_string` for text-based search when the online store supports it. + +`POST /retrieve-online-documents` is a deprecated alias with the same request body and response; new integrations should use `/search`. + +## [Alpha] OpenAI-Compatible Vector Store API + +{% hint style="warning" %} +**Alpha feature.** This API surface is functional and tested, but may change in future releases. +{% endhint %} + +The feature server exposes OpenAI-compatible vector store endpoints. This allows clients (including LLM agents and tool-calling frameworks) to discover and search vector data with plain text queries, without computing embeddings client-side. + +Each feature view with vector-indexed fields gets a deterministic `vs_{hash}` identifier derived from `SHA-256(project + ":" + feature_view_name)`. These IDs are stable across server restarts. + +### Endpoints + +| Method | Path | RBAC | Description | +|---|---|---|---| +| `GET` | `/v1/vector_stores` | `DESCRIBE` | List all vector stores (filtered by caller permissions) | +| `GET` | `/v1/vector_stores/{vector_store_id}` | `DESCRIBE` | Get metadata for a single vector store | +| `POST` | `/v1/vector_stores/{vector_store_id}/search` | `READ_ONLINE` | Search a vector store with server-side embedding | + +### Configuration + +Add an `embedding_model` section to your `feature_store.yaml`: + +```yaml +embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 +``` + +Feast uses **Sentence Transformers** (default) for local embedding inference — no external API key required. Custom embedding providers can be plugged in by implementing the `EmbeddingProvider` protocol. See [\[Alpha\] Vector Database](../alpha-vector-database.md#alpha-openai-compatible-vector-store-api) for full configuration, custom providers, filter details, and SDK usage. + ## Starting the feature server in TLS(SSL) mode Enabling TLS mode ensures that data between the Feast client and server is transmitted securely. For an ideal production environment, it is recommended to start the feature server in TLS mode. @@ -598,7 +634,11 @@ The [PyTorch NLP template](https://github.com/feast-dev/feast/tree/main/sdk/pyth | Endpoint | Resource Type | Permission | Description | |----------------------------|---------------------------------|-------------------------------------------------------|----------------------------------------------------------------| | /get-online-features | FeatureView,OnDemandFeatureView | Read Online | Get online features from the feature store | -| /retrieve-online-documents | FeatureView | Read Online | Retrieve online documents from the feature store for RAG | +| /search | FeatureView | Read Online | Vector similarity search for RAG (embedding vector or text query) | +| /retrieve-online-documents | FeatureView | Read Online | **Deprecated.** Use `/search` instead. | +| /v1/vector_stores | FeatureView | Describe | [Alpha] List all vector stores | +| /v1/vector_stores/{id} | FeatureView | Describe | [Alpha] Get a single vector store | +| /v1/vector_stores/{id}/search | FeatureView | Read Online | [Alpha] OpenAI-compatible vector search with server-side embedding | | /push | FeatureView | Write Online, Write Offline, Write Online and Offline | Push features to the feature store (online, offline, or both) | | /write-to-online-store | FeatureView | Write Online | Write features to the online store | | /materialize | FeatureView | Write Online | Materialize features within a specified time range | diff --git a/docs/reference/online-stores/scylladb.md b/docs/reference/online-stores/scylladb.md index 6c2e462877b..98dc03b24cb 100644 --- a/docs/reference/online-stores/scylladb.md +++ b/docs/reference/online-stores/scylladb.md @@ -111,6 +111,13 @@ result = store.retrieve_online_documents_v2( ) ``` +### Metadata filtering (OpenAI-compatible) + +ScyllaDB supports vector similarity search, but OpenAI-style metadata filtering is **not supported yet**. +Passing `filters` to `retrieve_online_documents_v2` or the OpenAI-compatible search endpoint raises `NotImplementedError`. + +For filtered vector search today, use one of the backends that implement metadata filters (for example Milvus, Elasticsearch, Postgres, SQLite, or MongoDB). See [Alpha Vector Database](../alpha-vector-database.md#supported-online-stores). + ## Functionality Matrix The set of functionality supported by online stores is described in detail [here](overview.md#functionality). diff --git a/infra/scripts/feature_server_docker_smoke.py b/infra/scripts/feature_server_docker_smoke.py index c8b3b440b7f..801decac90c 100644 --- a/infra/scripts/feature_server_docker_smoke.py +++ b/infra/scripts/feature_server_docker_smoke.py @@ -16,6 +16,7 @@ def list_projects(self, allow_cache=True, tags=None): class _FakeStore: def __init__(self): self.config = SimpleNamespace() + self.project = "smoke_test" self.registry = _FakeRegistry() self._provider = SimpleNamespace( async_supported=SimpleNamespace( @@ -32,6 +33,9 @@ async def initialize(self): def refresh_registry(self): return None + def list_feature_views(self): + return [] + async def close(self): return None diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2 index ccaadc29ff0..2c92401f83d 100644 --- a/infra/templates/README.md.jinja2 +++ b/infra/templates/README.md.jinja2 @@ -17,7 +17,7 @@ ## Join us on Slack! -👋👋👋 [Come say hi on Slack!](https://communityinviter.com/apps/feastopensource/feast-the-open-source-feature-store) +👋👋👋 [Come say hi on Slack!](https://slack.feast.dev/) [Check out our DeepWiki!](https://deepwiki.com/feast-dev/feast) diff --git a/infra/website/docs/blog/feast-agents-mcp.md b/infra/website/docs/blog/feast-agents-mcp.md index 5678cd5a578..bfa46ee8b02 100644 --- a/infra/website/docs/blog/feast-agents-mcp.md +++ b/infra/website/docs/blog/feast-agents-mcp.md @@ -51,7 +51,7 @@ feature_server: mcp_server_version: "1.0.0" ``` -Once enabled, any MCP-compatible agent -- whether built with LangChain, LlamaIndex, CrewAI, AutoGen, or a custom framework -- can connect to `http://your-feast-server/mcp` and discover available tools like `get-online-features` for entity-based retrieval, `retrieve-online-documents` for vector similarity search, and `write-to-online-store` for persisting agent state. +Once enabled, any MCP-compatible agent -- whether built with LangChain, LlamaIndex, CrewAI, AutoGen, or a custom framework -- can connect to `http://your-feast-server/mcp` and discover available tools like `get-online-features` for entity-based retrieval, `search` for vector similarity search, `vector_store_search` for OpenAI-compatible text search, and `write-to-online-store` for persisting agent state. ## A Concrete Example: Customer-Support Agent with Memory @@ -338,7 +338,7 @@ export OPENAI_BASE_URL="http://localhost:11434/v1" export LLM_MODEL="llama3.1:8b" ./run_demo.sh -# Any OpenAI-compatible provider (Azure, vLLM, LiteLLM, etc.) +# Any OpenAI-compatible provider (Azure, vLLM, etc.) export OPENAI_API_KEY="your-key" # pragma: allowlist secret export OPENAI_BASE_URL="https://your-endpoint/v1" export LLM_MODEL="your-model" diff --git a/infra/website/docs/blog/feast-openai-compatible-api.md b/infra/website/docs/blog/feast-openai-compatible-api.md new file mode 100644 index 00000000000..f228836afd6 --- /dev/null +++ b/infra/website/docs/blog/feast-openai-compatible-api.md @@ -0,0 +1,364 @@ +--- +title: "Using Feast's OpenAI Compatible Search API" +description: "Feast now exposes an OpenAI-compatible vector store search endpoint. Send a plain text query, get results back in the standard OpenAI format. No client-side embeddings required." +date: 2026-07-07 +authors: ["Chaitanya Patel", "Nikhil Kathole"] +--- + +
+ Sequence diagram showing a client sending a text query to Feast, which embeds and searches server-side +
+ +If you've tried to connect an AI agent to Feast's vector search, you've probably hit this wall: the agent needs to search your feature store, but Feast expects a raw embedding vector. The agent doesn't have one. It has a question in English. + +Until now, the workaround was ugly. You'd call an embedding provider (OpenAI, Ollama, whatever) to turn the text into a float array, then pass that array to Feast's vector search endpoint (`POST /search`, formerly `retrieve-online-documents`). Every client had to know both APIs, carry both sets of credentials, and run glue code whose only job was bridging the gap. + +Feast now has a new endpoint: `POST /v1/vector_stores/{vector_store_id}/search`. It follows the [OpenAI Vector Store Search API](https://platform.openai.com/docs/api-reference/vector-stores-search) format, including proper `vs_{hash}` identifiers for vector stores. You send text, Feast handles the embedding internally, and you get results back in the same JSON shape that OpenAI returns. No float arrays, no extra SDK. + +Each feature view with vector search enabled gets a deterministic `vs_` identifier (e.g. `vs_a1b2c3d4e5f6...`). Discover them via `GET /v1/vector_stores`. + +## The two-API tax + +Here's what searching Feast looked like before: + +```python +import openai +import requests + +# Step 1: Call the embedding provider yourself +embed_response = openai.embeddings.create( + model="text-embedding-3-small", + input="wireless noise-cancelling headphones" +) +query_vector = embed_response.data[0].embedding # 1536 floats + +# Step 2: Call Feast's proprietary API with the raw vector +result = requests.post("http://feast-server:6566/search", json={ + "features": [ + "product_catalog:vector", + "product_catalog:name", + "product_catalog:description", + "product_catalog:price", + ], + "query": query_vector, + "top_k": 5, + "api_version": 2, +}) +``` + +This works fine. But it has costs that add up: + +- Every service calling Feast needs an embedding SDK, an API key, and logic to handle the embedding call. Five microservices means five places managing embedding credentials. +- LLM agents can't use it. They discover tools through MCP or function calling, and they know how to call OpenAI-shaped endpoints. They don't know how to compute embeddings and pass raw float arrays to a custom API. +- The embedding model becomes a client-side decision. Different clients might use different models or versions, which means inconsistent search results against the same vector store. +- Feast's filter syntax is its own format. Not something an agent framework knows out of the box. + +## One endpoint, standard format + +With the new endpoint, that same search looks like this: + +```python +import requests + +# First, discover your vector store IDs +stores = requests.get("http://feast-server:6566/v1/vector_stores").json() +vs_id = stores["data"][0]["id"] # e.g. "vs_a1b2c3d4e5f6..." + +# Then search using the vs_ identifier +result = requests.post( + f"http://feast-server:6566/v1/vector_stores/{vs_id}/search", + json={ + "query": "wireless noise-cancelling headphones", + "max_num_results": 5, + }, +) +``` + +No embedding SDK. No raw vectors. The request and response match OpenAI's format, so anything that already talks to OpenAI can talk to Feast. + +### What happens under the hood + +When Feast receives this request, it: + +1. Embeds the query server-side using the model configured in `feature_store.yaml` (via [Sentence Transformers](https://www.sbert.net/) for local inference — no external API key required). +2. Runs vector similarity search against the feature view's online store (Postgres/pgvector, Milvus, Elasticsearch, SQLite, or whatever backend you've configured). +3. Applies filters if you provided any, using string equality, numeric comparisons, or compound AND/OR conditions in the OpenAI filter format. +4. Returns results in OpenAI's `vector_store.search_results.page` format. + +Because the embedding model is a server-side configuration, every client gets consistent results. No more worrying about whether service A is using `text-embedding-3-small` while service B accidentally stuck with `ada-002`. + +## Setting it up + +### Step 1: Configure the embedding model + +Add an `embedding_model` section to your `feature_store.yaml`: + +```yaml +project: my_project +registry: data/registry.db +provider: local + +online_store: + type: postgres + host: localhost + port: 5432 + database: feast + user: feast + password: ${DB_PASSWORD} + pgvector_enabled: true + vector_len: 384 + enable_openai_compatible_store: true + +embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 +``` + +Feast uses [Sentence Transformers](https://www.sbert.net/) for embedding, so everything runs locally — no external API key required. You can use any HuggingFace model compatible with `SentenceTransformer`: + +```yaml +# Default — lightweight, fast +embedding_model: + model: all-MiniLM-L6-v2 + +# Higher quality, larger model +embedding_model: + model: BAAI/bge-small-en-v1.5 +``` + +### Step 2: Define a feature view with vector search + +```python +from feast import Entity, FeatureView, Field +from feast.types import Array, Float32, String, Float64, Int64 +from datetime import timedelta + +product = Entity(name="product_id", join_keys=["product_id"]) + +product_catalog = FeatureView( + name="product_catalog", + entities=[product], + schema=[ + Field( + name="vector", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="COSINE", + ), + Field(name="name", dtype=String), + Field(name="description", dtype=String), + Field(name="category", dtype=String), + Field(name="price", dtype=Float64), + Field(name="rating", dtype=Float64), + ], + source=product_source, + ttl=timedelta(days=7), +) +``` + +### Step 3: Apply, load data, and serve + +```bash +feast apply +feast serve +``` + +### Step 4: Discover your vector store ID + +```bash +curl http://localhost:6566/v1/vector_stores +``` + +```json +{ + "object": "list", + "data": [ + { + "id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "object": "vector_store", + "name": "product_catalog", + "status": "completed", + "created_at": 1717200000 + } + ] +} +``` + +### Step 5: Search + +```bash +curl -X POST http://localhost:6566/v1/vector_stores/vs_a1b2c3d4e5f6a1b2c3d4e5f6/search \ + -H "Content-Type: application/json" \ + -d '{ + "query": "wireless noise-cancelling headphones", + "max_num_results": 3 + }' +``` + +Response: + +```json +{ + "object": "vector_store.search_results.page", + "search_query": ["wireless noise-cancelling headphones"], + "data": [ + { + "file_id": "vs_a1b2c3d4e5f6a1b2c3d4e5f6_42", + "filename": "vs_a1b2c3d4e5f6a1b2c3d4e5f6", + "score": 0.92, + "attributes": { + "name": "Sony WH-1000XM5", + "description": "Premium wireless noise-cancelling headphones", + "category": "Electronics", + "price": 349.99, + "rating": 4.8 + }, + "content": [ + {"type": "text", "text": "Sony WH-1000XM5"}, + {"type": "text", "text": "Premium wireless noise-cancelling headphones"}, + {"type": "text", "text": "Electronics"} + ] + } + ], + "has_more": false, + "next_page": null +} +``` + +The response follows OpenAI's `vector_store.search_results.page` schema. Any client that already parses OpenAI search results can parse this without changes. + +## Filtering + +The endpoint supports OpenAI-style filters for narrowing results beyond vector similarity. Filters work on the metadata stored alongside your vectors. + +### String filters + +```json +{ + "query": "running shoes", + "max_num_results": 5, + "filters": { + "type": "eq", + "key": "category", + "value": "Footwear" + } +} +``` + +### Numeric filters + +```json +{ + "query": "budget laptop", + "max_num_results": 5, + "filters": { + "type": "lt", + "key": "price", + "value": 500.0 + } +} +``` + +### Compound filters (AND / OR) + +```json +{ + "query": "wireless earbuds", + "max_num_results": 5, + "filters": { + "type": "and", + "filters": [ + {"type": "eq", "key": "category", "value": "Electronics"}, + {"type": "gte", "key": "rating", "value": 4.5}, + {"type": "lt", "key": "price", "value": 200.0} + ] + } +} +``` + +Comparison operators: `eq`, `ne`, `gt`, `gte`, `lt`, `lte`, `in`, `nin`. Compound operators: `and`, `or`. These nest to arbitrary depth. + +Numeric and boolean filters require the `enable_openai_compatible_store` flag in your online store config, plus a `feast apply` to add the `value_num` column to existing tables. String filters work on all existing schemas without migration. + +## What this means for AI agents + +We built this with agents in mind. When Feast added [MCP support](./feast-agents-mcp) earlier this year, agents could discover and call Feast tools dynamically. But vector search still had this gap where the agent needed to produce a float array. LLMs can't do that. + +Now the search tool is just text in, structured results out. An agent calls it the same way it calls any other OpenAI-compatible service. The feature server currently exposes these tools: + +| Capability | Endpoint | What it does | +|---|---|---| +| Structured feature lookup | `get-online-features` | Get customer profiles, account data, etc. | +| Vector search | `search` | Search with a pre-computed embedding vector (or text via `api_version: 2`) | +| List vector stores | `GET /v1/vector_stores` | Discover available vector stores and their `vs_` IDs | +| Get vector store | `GET /v1/vector_stores/{id}` | Get metadata for a specific vector store | +| Vector search (OpenAI format) | `POST /v1/vector_stores/{id}/search` | Search with plain text, embedding handled server-side | +| Write features / memory | `write-to-online-store` | Persist agent state, update features | + +`POST /retrieve-online-documents` remains available as a deprecated alias for `POST /search`. + +That last row is what this post is about. Before it existed, agents could read structured features and write state back, but they couldn't search vectors without help from glue code. + +## What this is, and what it isn't + +This makes Feast's vector search speak OpenAI's protocol. It doesn't turn Feast into a general purpose OpenAI-compatible vector database. + +| Works today | Not yet | +|---|---| +| `GET /v1/vector_stores` (list) | Creating vector stores via the API | +| `GET /v1/vector_stores/{id}` (get) | | +| `POST /v1/vector_stores/{id}/search` | | +| Plain text queries with server-side embedding | Client-provided embedding vectors on this endpoint | +| OpenAI-format filters (string, numeric, compound) | `ranking_options.score_threshold`, `ranking_options.ranker`, `rewrite_query: true` (rejected with 422) | +| All Feast online store backends | Standalone `/v1/embeddings` endpoint | + +Feature views are still defined in Python and managed through `feast apply`. Data is still ingested through Feast's existing write paths. The OpenAI-compatible layer is a read API that gives standard access to what's already in your feature store. + +## Deploying on Kubernetes + +Below is an example Kubernetes setup that deploys the feature server with Sentence Transformers for local embedding: + +```yaml +# configmap.yaml (embedding model section) +embedding_model: + provider: sentence_transformers + model: all-MiniLM-L6-v2 +``` + +```yaml +# deployment.yaml +containers: + - name: feast-server + command: ["feast", "serve", "-h", "0.0.0.0", "-p", "6566"] + ports: + - containerPort: 6566 +``` + +With this setup, embedding happens in-cluster. Nothing leaves your network. + +## Try it yourself + +```bash +# Install Feast with Sentence Transformers support +pip install feast sentence-transformers +``` + +Configure your `feature_store.yaml` with an `embedding_model` section, define a feature view with vector search enabled, run `feast apply`, load your data, start the server with `feast serve`, and search: + +```bash +# Discover your vector store IDs +curl -s http://localhost:6566/v1/vector_stores | python -m json.tool + +# Search using the vs_ identifier from the list response +curl -s http://localhost:6566/v1/vector_stores/YOUR_VS_ID/search \ + -H "Content-Type: application/json" \ + -d '{"query": "your search query", "max_num_results": 5}' | python -m json.tool +``` + +## What's next + +Next on the list: wiring up `ranking_options` and `rewrite_query` so they actually do something (right now they're accepted but ignored). We also want a standalone `/v1/embeddings` endpoint for clients that just need embeddings, and eventually the ability to create feature views through the OpenAI vector store API instead of requiring Python + `feast apply`. + +## Join the conversation + +If you're using this or have thoughts on what the OpenAI-compatible layer should support next, come find us on [Slack](https://slack.feast.dev/) or [GitHub](https://github.com/feast-dev/feast). diff --git a/infra/website/public/images/blog/feast-openai-compat-flow.png b/infra/website/public/images/blog/feast-openai-compat-flow.png new file mode 100644 index 00000000000..c7dcac8a5ed Binary files /dev/null and b/infra/website/public/images/blog/feast-openai-compat-flow.png differ diff --git a/protos/feast/core/SavedDataset.proto b/protos/feast/core/SavedDataset.proto index 111548aa480..c4a65accd36 100644 --- a/protos/feast/core/SavedDataset.proto +++ b/protos/feast/core/SavedDataset.proto @@ -48,6 +48,19 @@ message SavedDatasetSpec { // User defined metadata map tags = 7; + + // Optional logical namespace for hierarchical grouping. + // Maps to the top-level prefix in Iceberg REST Catalog API. + // Empty string means not set (no namespace scoping). + string namespace = 9; + + // Optional sub-grouping within a namespace. + // Maps to the namespace level in Iceberg REST Catalog API. + // Empty string means not set (dataset sits directly under namespace). + string collection = 10; + + // Description of the saved dataset. + string description = 11; } message SavedDatasetStorage { diff --git a/protos/feast/registry/RegistryServer.proto b/protos/feast/registry/RegistryServer.proto index cd60d47939f..2906e8a80b5 100644 --- a/protos/feast/registry/RegistryServer.proto +++ b/protos/feast/registry/RegistryServer.proto @@ -417,6 +417,10 @@ message ListSavedDatasetsRequest { map tags = 3; PaginationParams pagination = 4; SortingParams sorting = 5; + // Optional logical namespace filter. Empty string means no filter. + string namespace = 6; + // Optional collection filter. Empty string means no filter. + string collection = 7; } message ListSavedDatasetsResponse { diff --git a/sdk/python/feast/__init__.py b/sdk/python/feast/__init__.py index 30925e9fc4a..7be72ea6c86 100644 --- a/sdk/python/feast/__init__.py +++ b/sdk/python/feast/__init__.py @@ -19,13 +19,21 @@ from .data_source import KafkaSource, KinesisSource, PushSource, RequestSource from .dataframe import DataFrameEngine, FeastDataFrame from .doc_embedder import DocEmbedder, SchemaTransformFn -from .embedder import BaseEmbedder, EmbeddingConfig, MultiModalEmbedder +from .embedder import ( + BaseEmbedder, + EmbeddingConfig, + EmbeddingProvider, + MultiModalEmbedder, + SentenceTransformersEmbeddingProvider, + get_embedding_provider, +) from .entity import Entity from .feature import Feature from .feature_service import FeatureService from .feature_store import FeatureStore from .feature_view import FeatureView, FeatureViewState from .field import Field +from .filter_models import FilterTranslator from .labeling import ConflictPolicy, LabelView from .on_demand_feature_view import OnDemandFeatureView from .project import Project @@ -79,4 +87,8 @@ "BaseEmbedder", "MultiModalEmbedder", "EmbeddingConfig", + "EmbeddingProvider", + "SentenceTransformersEmbeddingProvider", + "get_embedding_provider", + "FilterTranslator", ] diff --git a/sdk/python/feast/api/registry/rest/rest_registry_server.py b/sdk/python/feast/api/registry/rest/rest_registry_server.py index bcd86984a95..b115d15aff7 100644 --- a/sdk/python/feast/api/registry/rest/rest_registry_server.py +++ b/sdk/python/feast/api/registry/rest/rest_registry_server.py @@ -78,12 +78,21 @@ def __init__(self, store: FeatureStore): registry_cfg = getattr(store.config, "registry", None) mcp_cfg = getattr(registry_cfg, "mcp", None) - if mcp_cfg and getattr(mcp_cfg, "enabled", False) is True: + if mcp_cfg and getattr(mcp_cfg, "enabled", False): try: from fastapi_mcp import FastApiMCP + from feast.infra.mcp_servers.mcp_server import ( + _patch_fastapi_mcp_schema_resolver, + ) + + _patch_fastapi_mcp_schema_resolver() mcp = FastApiMCP(self.app, name="feast-registry-mcp") - mcp.mount() + mount_sse = getattr(mcp, "mount_sse", None) + if mount_sse is not None: + mount_sse() + else: + mcp.mount() logger.info("MCP support enabled on REST registry server") except ImportError: logger.warning( diff --git a/sdk/python/feast/api/registry/rest/saved_datasets.py b/sdk/python/feast/api/registry/rest/saved_datasets.py index 4509c65f53d..ff6993183c2 100644 --- a/sdk/python/feast/api/registry/rest/saved_datasets.py +++ b/sdk/python/feast/api/registry/rest/saved_datasets.py @@ -43,6 +43,9 @@ class RegisterDatasetRequest(BaseModel): full_feature_names: bool = False feature_service_name: Optional[str] = None allow_override: bool = False + namespace: str = "" + collection: str = "" + description: str = "" class CreateDatasetRequest(BaseModel): @@ -75,10 +78,22 @@ def list_saved_datasets_all( limit: int = Query(50, ge=1, le=100), sort_by: str = Query(None), sort_order: str = Query("asc"), + namespace: Optional[str] = Query( + None, description="Filter by namespace (logical grouping)" + ), + collection: Optional[str] = Query( + None, description="Filter by collection (sub-grouping within namespace)" + ), include_relationships: bool = Query( False, description="Include relationships for each saved dataset" ), ): + extra_params = {} + if namespace is not None: + extra_params["namespace"] = namespace + if collection is not None: + extra_params["collection"] = collection + return aggregate_across_projects( grpc_handler=grpc_handler, list_method=grpc_handler.ListSavedDatasets, @@ -91,6 +106,7 @@ def list_saved_datasets_all( sort_by=sort_by, sort_order=sort_order, include_relationships=include_relationships, + extra_request_params=extra_params or None, ) @router.get("/saved_datasets/data/{name}") @@ -230,6 +246,12 @@ def list_saved_datasets( project: str = Query(...), allow_cache: bool = Query(default=True), tags: Dict[str, str] = Depends(parse_tags), + namespace: Optional[str] = Query( + None, description="Filter by namespace (logical grouping)" + ), + collection: Optional[str] = Query( + None, description="Filter by collection (sub-grouping within namespace)" + ), include_relationships: bool = Query( False, description="Include relationships for each saved dataset" ), @@ -240,6 +262,8 @@ def list_saved_datasets( project=project, allow_cache=allow_cache, tags=tags, + namespace=namespace or "", + collection=collection or "", pagination=create_grpc_pagination_params(pagination_params), sorting=create_grpc_sorting_params(sorting_params), ) @@ -347,6 +371,9 @@ def register_saved_dataset(payload: RegisterDatasetRequest = Body(...)): full_feature_names=payload.full_feature_names, storage=storage_proto, tags=payload.tags, + namespace=payload.namespace, + collection=payload.collection, + description=payload.description, ) if payload.feature_service_name: spec.feature_service_name = payload.feature_service_name diff --git a/sdk/python/feast/embedder.py b/sdk/python/feast/embedder.py index c847f78a21c..a0bccdd75ed 100644 --- a/sdk/python/feast/embedder.py +++ b/sdk/python/feast/embedder.py @@ -1,10 +1,128 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable, List, Optional +from typing import ( + TYPE_CHECKING, + Any, + Callable, + List, + Optional, + Protocol, + runtime_checkable, +) + +import pandas as pd if TYPE_CHECKING: import numpy as np -import pandas as pd + + from feast.repo_config import EmbeddingModelConfig + + +@runtime_checkable +class EmbeddingProvider(Protocol): + """Protocol for query-time embedding providers. + + Implement this to plug in any embedding backend (OpenAI, Cohere, + SentenceTransformers, a local model, etc.) for use in + ``openai_search`` and the OpenAI-compatible + feature server endpoints. + + Example:: + + class MyEmbeddingProvider: + def embed(self, texts: List[str]) -> List[List[float]]: + return my_model.encode(texts) + + async def aembed(self, texts: List[str]) -> List[List[float]]: + return await my_model.aencode(texts) + """ + + def embed(self, texts: List[str]) -> List[List[float]]: + """Return one embedding vector per input text.""" + ... + + async def aembed(self, texts: List[str]) -> List[List[float]]: + """Async variant of :meth:`embed`.""" + ... + + +class SentenceTransformersEmbeddingProvider: + """EmbeddingProvider backed by Sentence Transformers (local inference). + + Runs entirely on-device — no external API key required. Ideal for + air-gapped deployments, cost-sensitive workloads, or environments where + outbound API calls are restricted. + + Requires the ``sentence-transformers`` package:: + + pip install sentence-transformers + + Configuration in ``feature_store.yaml``:: + + embedding_model: + provider: sentence_transformers + model: all-MiniLM-L6-v2 # any HuggingFace sentence-transformers model + + The model is loaded lazily on the first call to :meth:`embed` or + :meth:`aembed` so that import cost is paid only when the provider is + actually used. + """ + + def __init__(self, model: str = "all-MiniLM-L6-v2"): + self.model_name = model + self._model = None + + @classmethod + def from_config( + cls, config: "EmbeddingModelConfig" + ) -> "SentenceTransformersEmbeddingProvider": + """Build a provider from a :class:`~feast.repo_config.EmbeddingModelConfig`.""" + return cls(model=config.model) + + def _load_model(self): + if self._model is None: + try: + from sentence_transformers import SentenceTransformer + except ImportError: + raise ImportError( + "sentence-transformers is required for the " + "'sentence_transformers' embedding provider. " + "Install with: pip install sentence-transformers" + ) + self._model = SentenceTransformer(self.model_name) + return self._model + + def embed(self, texts: List[str]) -> List[List[float]]: + model = self._load_model() + embeddings = model.encode(texts, convert_to_numpy=True) + return [emb.tolist() for emb in embeddings] + + async def aembed(self, texts: List[str]) -> List[List[float]]: + import asyncio + + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self.embed, texts) + + +def get_embedding_provider( + config: "EmbeddingModelConfig", +) -> "EmbeddingProvider": + """Factory that returns the appropriate :class:`EmbeddingProvider` for *config*. + + Dispatches on ``config.provider``: + + * ``"sentence_transformers"`` (default) → :class:`SentenceTransformersEmbeddingProvider` + + Raises: + ValueError: If ``config.provider`` is not a recognised value. + """ + provider = (config.provider or "sentence_transformers").lower() + if provider == "sentence_transformers": + return SentenceTransformersEmbeddingProvider.from_config(config) + raise ValueError( + f"Unknown embedding provider: '{config.provider}'. " + "Supported values: 'sentence_transformers'." + ) @dataclass diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index bba91130db3..1f374236790 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -49,10 +49,12 @@ from feast.data_source import PushMode from feast.errors import ( FeastError, + FeatureViewNotFoundException, ) from feast.feast_object import FeastObject from feast.feature_server_utils import convert_response_to_dict from feast.feature_view_utils import get_feature_view_from_feature_store +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.permissions.action import WRITE, AuthzedAction from feast.permissions.security_manager import ( assert_permissions, @@ -66,6 +68,7 @@ init_security_manager, str_to_auth_manager_type, ) +from feast.vector_store_utils import VectorStoreRegistry, build_vector_store_object # TODO: deprecate this in favor of push features @@ -114,7 +117,36 @@ class GetOnlineDocumentsRequest(BaseModel): top_k: Optional[int] = None query: Optional[List[float]] = None query_string: Optional[str] = None + distance_metric: Optional[str] = None api_version: Optional[int] = 1 + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None + + +class OpenAISearchMetadata(BaseModel): + features_to_retrieve: Optional[List[str]] = None + content_field: Optional[str] = None + + +class OpenAIRankingOptions(BaseModel): + ranker: Optional[str] = None + score_threshold: Optional[float] = None + + +class OpenAISearchRequest(BaseModel): + query: Union[str, List[str]] + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None + max_num_results: Optional[int] = 10 + ranking_options: Optional[OpenAIRankingOptions] = None + rewrite_query: Optional[bool] = None + metadata: Optional[OpenAISearchMetadata] = None + + +class OpenAIVectorStoreObject(BaseModel): + id: str + object: str = "vector_store" + name: str + status: str = "completed" + created_at: int = 0 class FeatureVectorResponse(BaseModel): @@ -324,7 +356,11 @@ def get_app( The app provides the following endpoints: - `/get-online-features`: Get online features - - `/retrieve-online-documents`: Retrieve online documents + - `/search`: Vector similarity search (RAG) + - `/retrieve-online-documents`: Deprecated alias for `/search` + - `/v1/vector_stores`: List vector stores (GET) + - `/v1/vector_stores/{vector_store_id}`: Get a vector store (GET) + - `/v1/vector_stores/{vector_store_id}/search`: OpenAI-compatible vector search - `/push`: Push features to the feature store - `/write-to-online-store`: Write to the online store - `/health`: Health check @@ -381,11 +417,14 @@ def stop_refresh(): if active_timer: active_timer.cancel() + vs_registry = VectorStoreRegistry(store) + def async_refresh(): if shutting_down: return store.refresh_registry() + vs_registry.refresh() if registry_ttl_sec: nonlocal active_timer @@ -456,18 +495,12 @@ async def get_online_features(request: GetOnlineFeaturesRequest) -> Any: ) return JSONResponse(content=response_dict) - @app.post( - "/retrieve-online-documents", - dependencies=[Depends(inject_user_details)], - response_model=OnlineFeaturesResponse, - ) - async def retrieve_online_documents( + async def _search_online_documents( request: GetOnlineDocumentsRequest, - ) -> Any: - with feast_metrics.track_request_latency("/retrieve-online-documents"): - logger.warning( - "This endpoint is in alpha and will be moved to /get-online-features when stable." - ) + *, + metrics_path: str, + ) -> JSONResponse: + with feast_metrics.track_request_latency(metrics_path): features = await _get_features(request, store) read_params = dict( @@ -477,6 +510,10 @@ async def retrieve_online_documents( ) if request.api_version == 2 and request.query_string is not None: read_params["query_string"] = request.query_string + if request.api_version == 2 and request.distance_metric is not None: + read_params["distance_metric"] = request.distance_metric + if request.api_version == 2 and request.filters is not None: + read_params["filters"] = request.filters if request.api_version == 2: read_params["include_feature_view_version_metadata"] = ( @@ -495,6 +532,124 @@ async def retrieve_online_documents( ) return JSONResponse(content=response_dict) + @app.post( + "/search", + dependencies=[Depends(inject_user_details)], + response_model=OnlineFeaturesResponse, + ) + async def search(request: GetOnlineDocumentsRequest) -> JSONResponse: + """Vector similarity search against online document embeddings.""" + return await _search_online_documents(request, metrics_path="/search") + + @app.post( + "/retrieve-online-documents", + dependencies=[Depends(inject_user_details)], + response_model=OnlineFeaturesResponse, + include_in_schema=False, + ) + async def retrieve_online_documents( + request: GetOnlineDocumentsRequest, + ) -> JSONResponse: + logger.warning( + "POST /retrieve-online-documents is deprecated; use POST /search instead." + ) + return await _search_online_documents( + request, metrics_path="/retrieve-online-documents" + ) + + @app.get( + "/v1/vector_stores", + dependencies=[Depends(inject_user_details)], + ) + async def list_vector_stores() -> JSONResponse: + permitted: list = [] + for obj in vs_registry.list_vector_stores(): + fv = vs_registry.resolve(obj["id"]) + try: + assert_permissions(resource=fv, actions=[AuthzedAction.DESCRIBE]) + permitted.append(obj) + except Exception: + pass + return JSONResponse(content={"object": "list", "data": permitted}) + + @app.get( + "/v1/vector_stores/{vector_store_id}", + dependencies=[Depends(inject_user_details)], + ) + async def get_vector_store(vector_store_id: str) -> JSONResponse: + try: + fv = vs_registry.resolve(vector_store_id) + assert_permissions(resource=fv, actions=[AuthzedAction.DESCRIBE]) + except FeatureViewNotFoundException: + return JSONResponse( + status_code=404, + content={ + "error": { + "message": f"No vector store found with id '{vector_store_id}'", + "type": "not_found_error", + } + }, + ) + return JSONResponse(content=build_vector_store_object(store.project, fv)) + + @app.post( + "/v1/vector_stores/{vector_store_id}/search", + dependencies=[Depends(inject_user_details)], + ) + async def vector_store_search( + vector_store_id: str, + request: OpenAISearchRequest, + ) -> JSONResponse: + with feast_metrics.track_request_latency( + "/v1/vector_stores/{vector_store_id}/search" + ): + try: + feature_view = vs_registry.resolve(vector_store_id) + assert_permissions( + resource=feature_view, + actions=[AuthzedAction.READ_ONLINE], + ) + + result = await store.openai_search( + vector_store_id=feature_view.name, + query=request.query, + vs_id=vector_store_id, + max_num_results=request.max_num_results or 10, + filters=request.filters, + ranking_options=( + request.ranking_options.model_dump() + if request.ranking_options + else None + ), + rewrite_query=request.rewrite_query, + features_to_retrieve=( + request.metadata.features_to_retrieve + if request.metadata + else None + ), + ) + except FeatureViewNotFoundException: + return JSONResponse( + status_code=404, + content={ + "error": { + "message": f"No vector store found with id '{vector_store_id}'", + "type": "not_found_error", + } + }, + ) + except ValueError as e: + return JSONResponse( + status_code=422, + content={ + "error": { + "message": str(e), + "type": "invalid_request_error", + } + }, + ) + return JSONResponse(content=result) + @app.post("/push", dependencies=[Depends(inject_user_details)]) async def push(request: PushFeaturesRequest) -> Response: with feast_metrics.track_request_latency("/push"): diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index ba110c1e0a4..c9888768003 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -38,6 +38,7 @@ if TYPE_CHECKING: from feast.diff.apply_progress import ApplyProgressContext + from feast.embedder import EmbeddingProvider import pandas as pd import pyarrow as pa @@ -72,10 +73,12 @@ from feast.feature_service import FeatureService from feast.feature_view import ( DUMMY_ENTITY, + DUMMY_ENTITY_ID, DUMMY_ENTITY_NAME, FeatureView, FeatureViewState, ) +from feast.filter_models import ComparisonFilter, CompoundFilter, convert_dict_to_filter from feast.inference import ( update_data_sources_with_inferred_event_timestamp_col, update_feature_views_with_inferred_features_and_entities, @@ -107,7 +110,12 @@ from feast.stream_feature_view import StreamFeatureView from feast.transformation.pandas_transformation import PandasTransformation from feast.transformation.python_transformation import PythonTransformation -from feast.utils import _get_feature_view_vector_field_metadata, _utc_now +from feast.utils import ( + _distance_to_score, + _get_feature_view_vector_field_metadata, + _utc_now, +) +from feast.vector_store_utils import feature_view_to_vs_id from feast.version_utils import parse_version try: @@ -170,6 +178,7 @@ class FeatureStore: _registry: Optional[BaseRegistry] _provider: Optional[Provider] _openlineage_emitter: Optional[Any] = None + _embedding_provider: Optional["EmbeddingProvider"] _feature_service_cache: Dict[str, List[str]] def __init__( @@ -177,6 +186,7 @@ def __init__( repo_path: Optional[str] = None, config: Optional[RepoConfig] = None, fs_yaml_file: Optional[Path] = None, + embedding_provider: Optional["EmbeddingProvider"] = None, ): """ Creates a FeatureStore object. @@ -186,6 +196,10 @@ def __init__( config (optional): Configuration object used to configure the feature store. fs_yaml_file (optional): Path to the `feature_store.yaml` file used to configure the feature store. At most one of 'fs_yaml_file' and 'config' can be set. + embedding_provider (optional): Custom embedding provider implementing + the :class:`~feast.embedder.EmbeddingProvider` protocol. When not + supplied, a :class:`~feast.embedder.SentenceTransformersEmbeddingProvider` is + created from ``embedding_model`` in ``feature_store.yaml``. Raises: ValueError: If both or neither of repo_path and config are specified. @@ -218,6 +232,7 @@ def __init__( self._current_project: ContextVar[Optional[str]] = ContextVar( "current_project", default=None ) + self._embedding_provider = embedding_provider # Initialize feature service cache for performance optimization self._feature_service_cache = {} @@ -382,6 +397,31 @@ def __repr__(self) -> str: f")" ) + @property + def embedding_provider(self) -> "EmbeddingProvider": + """Return the active embedding provider, creating one from config if needed.""" + if self._embedding_provider is None: + from feast.embedder import get_embedding_provider + + embed_cfg = self.config.embedding_model + if embed_cfg is None: + raise ValueError( + "No embedding provider set and embedding_model is not " + "configured in feature_store.yaml. Either pass an " + "embedding_provider to FeatureStore() or add an " + "'embedding_model' section to feature_store.yaml.\n" + "Example:\n" + " embedding_model:\n" + " provider: sentence_transformers\n" + " model: all-MiniLM-L6-v2" + ) + self._embedding_provider = get_embedding_provider(embed_cfg) + return self._embedding_provider + + @embedding_provider.setter + def embedding_provider(self, provider: "EmbeddingProvider") -> None: + self._embedding_provider = provider + @property def registry(self) -> BaseRegistry: """Gets the registry of this feature store.""" @@ -3943,6 +3983,7 @@ def retrieve_online_documents_v2( image_weight: float = 0.5, combine_strategy: str = "weighted_sum", include_feature_view_version_metadata: bool = False, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, ) -> OnlineResponse: """ Retrieves the top k closest document features. Note, embeddings are a subset of features. @@ -4097,9 +4138,182 @@ def retrieve_online_documents_v2( top_k, distance_metric, query_string, + filters, include_feature_view_version_metadata, ) + async def openai_search( + self, + vector_store_id: str, + query: Union[str, List[str]], + *, + vs_id: Optional[str] = None, + max_num_results: int = 10, + filters: Optional[ + Union[ComparisonFilter, CompoundFilter, Dict[str, Any]] + ] = None, + ranking_options: Optional[Dict[str, Any]] = None, + rewrite_query: Optional[bool] = None, + features_to_retrieve: Optional[List[str]] = None, + ) -> Dict[str, Any]: + """ + OpenAI-compatible vector store search. + + Accepts a raw query string, embeds it via the configured embedding + provider (when ``embedding_model`` is configured in feature_store.yaml), + and returns results in OpenAI's ``vector_store.search_results.page`` + format. + + Args: + vector_store_id: Feature view name (maps to the OpenAI + ``vector_store_id`` path parameter). + query: Natural language query string, or list of strings. + max_num_results: Maximum number of results to return. + filters: OpenAI-compatible filters applied to the search. + ranking_options: OpenAI-compatible ranking options. Currently + unsupported; a ``ValueError`` is raised if + ``score_threshold`` or ``ranker`` are provided. + rewrite_query: Whether to rewrite the query. Currently + unsupported; ``False`` (the default/no-op) is accepted, + but ``True`` raises a ``ValueError``. + features_to_retrieve: Specific feature names to return. + If None, all features from the feature view are used. + + Returns: + Dict matching the OpenAI ``vector_store.search_results.page`` + schema. + + Examples: + Keyword search (no embedding model configured):: + + result = await store.openai_search( + vector_store_id="city_embeddings", + query="cities in California", + max_num_results=5, + ) + + Vector search (embedding model configured in YAML):: + + # feature_store.yaml has: + # embedding_model: + # model: text-embedding-3-small + result = await store.openai_search( + vector_store_id="product_embeddings", + query="wireless audio device", + max_num_results=3, + features_to_retrieve=["name", "description"], + ) + """ + unsupported: List[str] = [] + if ranking_options: + if ranking_options.get("score_threshold") is not None: + unsupported.append("ranking_options.score_threshold") + if ranking_options.get("ranker") is not None: + unsupported.append("ranking_options.ranker") + if rewrite_query is True: + unsupported.append("rewrite_query") + if unsupported: + raise ValueError( + f"The following parameters are not yet supported: " + f"{', '.join(unsupported)}. Remove them from the request or " + f"wait for a future release that implements them." + ) + + feature_view = self.get_feature_view(vector_store_id) + display_id = vs_id or feature_view_to_vs_id(self.project, vector_store_id) + + vector_field_metadata = _get_feature_view_vector_field_metadata(feature_view) + distance_metric: Optional[str] = None + if vector_field_metadata and vector_field_metadata.vector_search_metric: + distance_metric = vector_field_metadata.vector_search_metric + + if features_to_retrieve: + feature_names = features_to_retrieve + else: + feature_names = [ + f.name for f in feature_view.features if not f.vector_index + ] + + features = [f"{feature_view.name}:{name}" for name in feature_names] + query_text = query if isinstance(query, str) else " ".join(query) + + embeddings = await self.embedding_provider.aembed([query_text]) + query_embedding = embeddings[0] + + typed_filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None + if filters is not None: + if isinstance(filters, dict): + typed_filters = convert_dict_to_filter(filters) + else: + typed_filters = filters + + response = await run_in_threadpool( + lambda: self.retrieve_online_documents_v2( + features=features, + query=query_embedding, + top_k=max_num_results, + filters=typed_filters, + ) + ) + + response_dict = response.to_dict() + + entity_key_names = { + col.name + for col in feature_view.entity_columns + if col.name != DUMMY_ENTITY_ID + } + + result_data = [] + if response_dict: + first_key = next(iter(response_dict)) + num_rows = len(response_dict.get(first_key, [])) + for i in range(num_rows): + score = 0.0 + attributes: Dict[str, Any] = {} + content_parts: List[Dict[str, str]] = [] + + for key, values in response_dict.items(): + val = values[i] if i < len(values) else None + if key == "distance": + raw = float(val) if val is not None else 0.0 + score = _distance_to_score(raw, distance_metric) + elif key not in entity_key_names: + attributes[key] = val + if isinstance(val, str): + content_parts.append({"type": "text", "text": val}) + + if entity_key_names: + key_parts = [ + str(response_dict[k][i]) + for k in sorted(entity_key_names) + if k in response_dict and i < len(response_dict[k]) + ] + file_id = f"{display_id}_{'_'.join(key_parts)}" + else: + file_id = f"{display_id}_{i}" + + result_data.append( + { + "file_id": file_id, + "filename": display_id, + "score": score, + "attributes": attributes, + "content": content_parts + if content_parts + else [{"type": "text", "text": str(attributes)}], + } + ) + + search_query = query if isinstance(query, list) else [query] + return { + "object": "vector_store.search_results.page", + "search_query": search_query, + "data": result_data, + "has_more": False, + "next_page": None, + } + def _retrieve_from_online_store( self, provider: Provider, @@ -4162,23 +4376,25 @@ def _retrieve_from_online_store_v2( top_k: int, distance_metric: Optional[str], query_string: Optional[str], + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> OnlineResponse: """ Search and return document features from the online document store. """ vector_field_metadata = _get_feature_view_vector_field_metadata(table) - if vector_field_metadata: + if vector_field_metadata and vector_field_metadata.vector_search_metric: distance_metric = vector_field_metadata.vector_search_metric documents = provider.retrieve_online_documents_v2( config=self.config, table=table, requested_features=requested_features, - query=query, + embedding=query, top_k=top_k, distance_metric=distance_metric, query_string=query_string, + filters=filters, include_feature_view_version_metadata=include_feature_view_version_metadata, ) @@ -4554,7 +4770,11 @@ def delete_project(self, name: str, commit: bool = True) -> None: return self.registry.delete_project(name, commit=commit) def list_saved_datasets( - self, allow_cache: bool = False, tags: Optional[dict[str, str]] = None + self, + allow_cache: bool = False, + tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: """ Retrieves the list of saved datasets from the registry. @@ -4562,12 +4782,18 @@ def list_saved_datasets( Args: allow_cache: Whether to allow returning saved datasets from a cached registry. tags: Filter by tags. + namespace: Filter by logical namespace grouping. + collection: Filter by collection sub-grouping within namespace. Returns: A list of saved datasets. """ return self.registry.list_saved_datasets( - self.project, allow_cache=allow_cache, tags=tags + self.project, + allow_cache=allow_cache, + tags=tags, + namespace=namespace, + collection=collection, ) async def initialize(self) -> None: diff --git a/sdk/python/feast/filter_models.py b/sdk/python/feast/filter_models.py new file mode 100644 index 00000000000..4b98d0034f9 --- /dev/null +++ b/sdk/python/feast/filter_models.py @@ -0,0 +1,101 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel + + +class ComparisonFilter(BaseModel): + """A filter that compares a metadata field against a value. + + :param type: The comparison operator to apply + :param key: The metadata field name to filter on + :param value: The value to compare against + """ + + type: Literal["eq", "ne", "gt", "gte", "lt", "lte", "in", "nin"] + key: str + value: Any + + +class CompoundFilter(BaseModel): + """A filter that combines multiple filters with a logical operator. + + :param type: The logical operator ("and" requires all filters match, + "or" requires any filter matches) + :param filters: The list of filters to combine + """ + + type: Literal["and", "or"] + filters: List[Union[ComparisonFilter, "CompoundFilter"]] + + +CompoundFilter.model_rebuild() + +FilterType = Optional[Union[ComparisonFilter, CompoundFilter]] + + +class FilterTranslator(ABC): + """Abstract base for translating Feast filters into backend-native expressions. + + Each online store backend provides a concrete subclass that knows how to + turn :class:`ComparisonFilter` / :class:`CompoundFilter` trees into the + query language understood by the underlying database or search engine. + + The :meth:`translate` entry-point handles ``None`` via :meth:`_empty` and + delegates to :meth:`translate_comparison` / :meth:`translate_compound` + through the shared :meth:`_dispatch` recursion, so subclasses only need to + implement the three abstract leaf methods. + """ + + @abstractmethod + def translate(self, filters: FilterType) -> Any: + """Return the backend-native filter expression.""" + ... + + def _dispatch(self, filter_obj: Union[ComparisonFilter, CompoundFilter]) -> Any: + """Route to comparison or compound handler (used for recursion).""" + if isinstance(filter_obj, ComparisonFilter): + return self.translate_comparison(filter_obj) + elif isinstance(filter_obj, CompoundFilter): + return self.translate_compound(filter_obj) + raise ValueError(f"Unknown filter type: {type(filter_obj)}") + + @abstractmethod + def translate_comparison(self, f: ComparisonFilter) -> Any: + """Translate a single comparison into a backend-native expression.""" + ... + + @abstractmethod + def translate_compound(self, f: CompoundFilter) -> Any: + """Translate a compound (AND/OR) filter into a backend-native expression.""" + ... + + +def filters_contain_numeric_comparison( + filter_obj: Union[ComparisonFilter, CompoundFilter], +) -> bool: + """Return True if any leaf comparison uses a numeric value.""" + if isinstance(filter_obj, ComparisonFilter): + return isinstance(filter_obj.value, (int, float)) and not isinstance( + filter_obj.value, bool + ) + if isinstance(filter_obj, CompoundFilter): + return any(filters_contain_numeric_comparison(f) for f in filter_obj.filters) + return False + + +def convert_dict_to_filter( + filter_dict: Dict[str, Any], +) -> Union[ComparisonFilter, CompoundFilter]: + """Convert a raw dict (e.g. from OpenAI-compatible JSON) into a typed filter object.""" + filter_type = filter_dict.get("type") + if filter_type in ("and", "or"): + return CompoundFilter( + type=filter_type, + filters=[convert_dict_to_filter(f) for f in filter_dict["filters"]], + ) + return ComparisonFilter( + type=filter_dict["type"], + key=filter_dict["key"], + value=filter_dict["value"], + ) diff --git a/sdk/python/feast/infra/mcp_servers/mcp_server.py b/sdk/python/feast/infra/mcp_servers/mcp_server.py index 972023cdd12..2be788148cd 100644 --- a/sdk/python/feast/infra/mcp_servers/mcp_server.py +++ b/sdk/python/feast/infra/mcp_servers/mcp_server.py @@ -6,9 +6,10 @@ """ import logging -from typing import Optional +from typing import TYPE_CHECKING, Any, Dict, Optional, Set -from feast.feature_store import FeatureStore +if TYPE_CHECKING: + from feast.feature_store import FeatureStore logger = logging.getLogger(__name__) @@ -30,13 +31,84 @@ class McpTransportNotSupportedError(RuntimeError): pass -def add_mcp_support_to_app(app, store: FeatureStore, config) -> Optional["FastApiMCP"]: +def _resolve_schema_references_safe( + schema_part: Dict[str, Any], + reference_schema: Dict[str, Any], + _seen_refs: Optional[Set[str]] = None, +) -> Dict[str, Any]: + """Resolve ``$ref`` pointers in an OpenAPI schema **without** infinite recursion. + + ``fastapi_mcp`` <=0.4.0 ships a ``resolve_schema_references`` helper that + inlines every ``$ref`` it encounters, but never tracks which refs have + already been visited. Feast's OpenAPI schema contains self-referential + types (protobuf ``Value`` -> ``Struct`` -> ``Value``), which causes a + ``RecursionError``. + + This replacement keeps a *seen-refs* set and replaces any circular + ``$ref`` with an empty object schema instead of recursing forever. + + Reference: ``fastapi_mcp/openapi/utils.py`` lines 19-55 + https://github.com/tadata-org/fastapi_mcp/blob/main/fastapi_mcp/openapi/utils.py#L19-L55 + """ + if _seen_refs is None: + _seen_refs = set() + + schema_part = schema_part.copy() + + if "$ref" in schema_part: + ref_path = schema_part["$ref"] + if ref_path in _seen_refs: + schema_part.pop("$ref") + schema_part.setdefault("type", "object") + return schema_part + if ref_path.startswith("#/components/schemas/"): + model_name = ref_path.split("/")[-1] + schemas = reference_schema.get("components", {}).get("schemas", {}) + if model_name in schemas: + _seen_refs = _seen_refs | {ref_path} + ref_schema = schemas[model_name].copy() + schema_part.pop("$ref") + schema_part.update(ref_schema) + + for key, value in schema_part.items(): + if isinstance(value, dict): + schema_part[key] = _resolve_schema_references_safe( + value, reference_schema, _seen_refs + ) + elif isinstance(value, list): + schema_part[key] = [ + _resolve_schema_references_safe(item, reference_schema, _seen_refs) + if isinstance(item, dict) + else item + for item in value + ] + + return schema_part + + +def _patch_fastapi_mcp_schema_resolver() -> None: + """Monkey-patch ``fastapi_mcp.openapi.utils.resolve_schema_references`` + with our circular-ref-safe version so that ``FastApiMCP`` can process + Feast's OpenAPI schema without hitting a ``RecursionError``.""" + try: + import fastapi_mcp.openapi.utils as _mcp_utils + + _mcp_utils.resolve_schema_references = _resolve_schema_references_safe # type: ignore[assignment] + except (ImportError, AttributeError): + pass + + +def add_mcp_support_to_app( + app, store: "FeatureStore", config +) -> Optional["FastApiMCP"]: """Add MCP support to the FastAPI app if enabled in configuration.""" if not MCP_AVAILABLE: logger.warning("MCP support requested but fastapi_mcp is not available") return None try: + _patch_fastapi_mcp_schema_resolver() + # Create MCP server from the FastAPI app mcp = FastApiMCP( app, diff --git a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py index b78d003ac25..e29ba9df49a 100644 --- a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py +++ b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py @@ -5,11 +5,18 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Union from elasticsearch import Elasticsearch, helpers from feast import Entity, FeatureView, RepoConfig +from feast.filter_models import ( + ComparisonFilter, + CompoundFilter, + FilterTranslator, + FilterType, + filters_contain_numeric_comparison, +) from feast.infra.key_encoding_utils import ( deserialize_entity_key, get_list_val_str, @@ -45,10 +52,106 @@ class ElasticSearchOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): # The number of rows to write in a single batch write_batch_size: Optional[int] = 40 + enable_openai_compatible_store: Optional[bool] = False + + +logger = logging.getLogger(__name__) + + +class ElasticsearchFilterTranslator(FilterTranslator): + """Translates Feast filters into Elasticsearch Query DSL clauses.""" + + def __init__(self, has_value_num: bool = False): + self.has_value_num = has_value_num + + def translate(self, filters: FilterType) -> List[Dict[str, Any]]: + if filters is None: + return [] + return [self._dispatch(filters)] + + def translate_comparison(self, f: ComparisonFilter) -> Dict[str, Any]: + is_numeric = isinstance(f.value, (int, float)) and not isinstance(f.value, bool) + is_numeric_list = ( + isinstance(f.value, list) + and f.value + and isinstance(f.value[0], (int, float)) + and not isinstance(f.value[0], bool) + ) + + if self.has_value_num and (is_numeric or is_numeric_list): + field = f"{f.key}.value_num" + exact_field = field + fmt_val = f.value + fmt_list = f.value if is_numeric_list else None + else: + exact_field = f"{f.key}.value_text.keyword" + field = f"{f.key}.value_text" + fmt_val = str(f.value) + fmt_list = [str(v) for v in f.value] if isinstance(f.value, list) else None + + if f.type == "eq": + return {"term": {exact_field: fmt_val}} + elif f.type == "ne": + return {"bool": {"must_not": [{"term": {exact_field: fmt_val}}]}} + elif f.type in ("gt", "gte", "lt", "lte"): + return {"range": {field: {f.type: fmt_val}}} + elif f.type == "in": + if not isinstance(f.value, list): + raise ValueError( + f"'in' filter requires a list value, got {type(f.value)}" + ) + return {"terms": {exact_field: fmt_list}} + elif f.type == "nin": + if not isinstance(f.value, list): + raise ValueError( + f"'nin' filter requires a list value, got {type(f.value)}" + ) + return {"bool": {"must_not": [{"terms": {exact_field: fmt_list}}]}} + raise ValueError(f"Unsupported comparison operator: {f.type}") + + def translate_compound(self, f: CompoundFilter) -> Dict[str, Any]: + clauses = [self._dispatch(sub) for sub in f.filters] + if f.type == "and": + return {"bool": {"must": clauses}} + else: + return {"bool": {"should": clauses, "minimum_should_match": 1}} class ElasticSearchOnlineStore(OnlineStore): _client: Optional[Elasticsearch] = None + _index_value_num_cache: Optional[Dict[str, bool]] = None + + def _index_has_value_num(self, config: RepoConfig, index_name: str) -> bool: + """Check the actual ES index mapping for the value_num field. + + Caches the result per index so we only hit ES once. + """ + if self._index_value_num_cache is None: + self._index_value_num_cache = {} + if index_name in self._index_value_num_cache: + return self._index_value_num_cache[index_name] + try: + mapping = self._get_client(config).indices.get_mapping(index=index_name) + templates = ( + mapping.get(index_name, {}) + .get("mappings", {}) + .get("dynamic_templates", []) + ) + for tmpl in templates: + for _, tmpl_body in tmpl.items(): + props = tmpl_body.get("mapping", {}).get("properties", {}) + if "value_num" in props: + self._index_value_num_cache[index_name] = True + return True + except Exception as e: + logging.warning( + "Failed to check index mapping for value_num on '%s': %s: %s", + index_name, + type(e).__name__, + e, + ) + self._index_value_num_cache[index_name] = False + return False def _get_client(self, config: RepoConfig) -> Elasticsearch: online_store_config = config.online_store @@ -94,6 +197,7 @@ def online_write_batch( progress: Optional[Callable[[int], Any]], ) -> None: insert_values = [] + include_value_num = self._index_has_value_num(config, table.name) grouped_docs: dict[str, dict[str, Any]] = defaultdict( lambda: { "features": {}, @@ -115,7 +219,7 @@ def online_write_batch( doc_key = f"{encoded_entity_key}_{timestamp}" for feature_name, value in values.items(): - doc = _encode_feature_value(value) + doc = _encode_feature_value(value, include_value_num=include_value_num) grouped_docs[doc_key]["features"][feature_name] = doc grouped_docs[doc_key]["timestamp"] = timestamp grouped_docs[doc_key]["created_ts"] = created_ts @@ -209,6 +313,23 @@ def create_index(self, config: RepoConfig, table: FeatureView): _get_feature_view_vector_field_metadata(table), "vector_length", 512 ) + feature_properties: Dict[str, Any] = { + "feature_value": {"type": "binary"}, + "value_text": { + "type": "text", + "fields": {"keyword": {"type": "keyword", "ignore_above": 256}}, + }, + "vector_value": { + "type": "dense_vector", + "dims": vector_field_length, + "index": True, + "similarity": config.online_store.similarity, + }, + } + + if getattr(config.online_store, "enable_openai_compatible_store", False): + feature_properties["value_num"] = {"type": "double"} + index_mapping = { "dynamic_templates": [ { @@ -217,16 +338,7 @@ def create_index(self, config: RepoConfig, table: FeatureView): "match": "*", "mapping": { "type": "object", - "properties": { - "feature_value": {"type": "binary"}, - "value_text": {"type": "text"}, - "vector_value": { - "type": "dense_vector", - "dims": vector_field_length, - "index": True, - "similarity": config.online_store.similarity, - }, - }, + "properties": feature_properties, }, } } @@ -348,6 +460,7 @@ def retrieve_online_documents_v2( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -383,6 +496,27 @@ def retrieve_online_documents_v2( source_fields += composite_key_name body["_source"] = source_fields + has_value_num = self._index_has_value_num(config, es_index) + + if ( + filters + and filters_contain_numeric_comparison(filters) + and not has_value_num + ): + logger.warning( + "Numeric comparison filters (gt, gte, lt, lte) are being used " + "but this index does not have a 'value_num' field. Numeric " + "fields are stored as text, which causes lexicographic " + "comparison instead of numeric comparison (e.g. '9' > '100'). " + "To fix this, set 'enable_openai_compatible_store: true' in " + "your online_store config, then teardown and re-apply your " + "feature store to recreate indices with the value_num field." + ) + + metadata_filters = ElasticsearchFilterTranslator(has_value_num).translate( + filters + ) + if embedding: similarity = (distance_metric or config.online_store.similarity).lower() vector_field_path = ( @@ -399,37 +533,47 @@ def retrieve_online_documents_v2( f"Unsupported similarity/distance_metric: {similarity}" ) - # Hybrid search if embedding and query_string: + bool_clause: Dict[str, Any] = { + "must": [ + {"query_string": {"query": f'"{query_string}"'}}, + {"exists": {"field": vector_field_path}}, + ] + } + if metadata_filters: + bool_clause["filter"] = metadata_filters body["query"] = { "script_score": { - "query": { - "bool": { - "must": [ - {"query_string": {"query": f'"{query_string}"'}}, - {"exists": {"field": vector_field_path}}, - ] - } - }, + "query": {"bool": bool_clause}, "script": { "source": script, "params": {"query_vector": embedding}, }, } } - # Vector search only elif embedding: + filter_clauses: List[Dict[str, Any]] = [ + {"exists": {"field": vector_field_path}} + ] + filter_clauses.extend(metadata_filters) body["query"] = { "script_score": { - "query": { - "bool": {"filter": [{"exists": {"field": vector_field_path}}]} - }, + "query": {"bool": {"filter": filter_clauses}}, "script": {"source": script, "params": {"query_vector": embedding}}, } } - # Keyword search only elif query_string: - body["query"] = {"query_string": {"query": f'"{query_string}"'}} + if metadata_filters: + body["query"] = { + "bool": { + "must": [ + {"query_string": {"query": f'"{query_string}"'}}, + ], + "filter": metadata_filters, + } + } + else: + body["query"] = {"query_string": {"query": f'"{query_string}"'}} response = self._get_client(config).search(index=es_index, body=body) @@ -443,7 +587,6 @@ def retrieve_online_documents_v2( timestamp = row["_source"]["timestamp"] timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f") - # Create feature dict with all requested features feature_dict = {"distance": _to_value_proto(float(row["_score"]))} if query_string is not None: feature_dict["text_rank"] = _to_value_proto(float(row["_score"])) @@ -468,14 +611,14 @@ def _to_value_proto(value: Any) -> ValueProto: val_proto = ValueProto() if isinstance(value, ValueProto): return value - if isinstance(value, float): + if isinstance(value, bool): + val_proto.bool_val = value + elif isinstance(value, float): val_proto.float_val = value elif isinstance(value, str): val_proto.string_val = value elif isinstance(value, int): val_proto.int64_val = value - elif isinstance(value, bool): - val_proto.bool_val = value elif isinstance(value, list) and all(isinstance(v, float) for v in value): val_proto.float_list_val.val.extend(value) elif isinstance(value, dict) and "feature_value" in value: @@ -489,12 +632,15 @@ def _to_value_proto(value: Any) -> ValueProto: return val_proto -def _encode_feature_value(value: ValueProto) -> Dict[str, Any]: +def _encode_feature_value( + value: ValueProto, + include_value_num: bool = False, +) -> Dict[str, Any]: """ Encode a ValueProto into a dictionary for Elasticsearch storage. """ encoded_value = base64.b64encode(value.SerializeToString()).decode("utf-8") - result = {"feature_value": encoded_value} + result: Dict[str, Any] = {"feature_value": encoded_value} vector_val = get_list_val_str(value) if vector_val: @@ -505,8 +651,24 @@ def _encode_feature_value(value: ValueProto) -> Dict[str, Any]: result["value_text"] = value.bytes_val.decode("utf-8") elif value.HasField("int64_val"): result["value_text"] = str(value.int64_val) + if include_value_num: + result["value_num"] = value.int64_val + elif value.HasField("int32_val"): + result["value_text"] = str(value.int32_val) + if include_value_num: + result["value_num"] = value.int32_val elif value.HasField("double_val"): result["value_text"] = str(value.double_val) + if include_value_num: + result["value_num"] = value.double_val + elif value.HasField("float_val"): + result["value_text"] = str(value.float_val) + if include_value_num: + result["value_num"] = value.float_val + elif value.HasField("bool_val"): + result["value_text"] = str(value.bool_val) + if include_value_num: + result["value_num"] = 1.0 if value.bool_val else 0.0 return result diff --git a/sdk/python/feast/infra/online_stores/helpers.py b/sdk/python/feast/infra/online_stores/helpers.py index 59ef9185c1f..eed47fe7837 100644 --- a/sdk/python/feast/infra/online_stores/helpers.py +++ b/sdk/python/feast/infra/online_stores/helpers.py @@ -1,6 +1,6 @@ import struct from datetime import datetime, timezone -from typing import Any, List +from typing import Any, List, Optional, Tuple import mmh3 @@ -72,6 +72,27 @@ def _to_naive_utc(ts: datetime) -> datetime: return ts.astimezone(tz=timezone.utc).replace(tzinfo=None) +def extract_text_and_num( + val: Any, compute_num: bool +) -> Tuple[Optional[str], Optional[float]]: + """Extract (value_text, value_num) from a ValueProto. + + Used by SQL-based online stores to populate the value_text and optional + value_num columns without duplicating type-dispatch logic. + """ + val_type = val.WhichOneof("val") + if val_type == "string_val": + return val.string_val, None + if val_type in ("int64_val", "int32_val", "double_val", "float_val"): + raw = getattr(val, val_type) + return str(raw), float(raw) if compute_num else None + if val_type == "bool_val": + return str(val.bool_val), ( + 1.0 if val.bool_val else 0.0 + ) if compute_num else None + return None, None + + def compute_versioned_name(table: Any, enable_versioning: bool = False) -> str: """Return the table name with a ``_v{N}`` suffix when versioning is enabled.""" name = table.name diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py index 19667de8c4b..00079938a86 100644 --- a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -1,5 +1,6 @@ import base64 import logging +import re from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union @@ -14,6 +15,13 @@ from feast import Entity from feast.feature_view import FeatureView +from feast.filter_models import ( + ComparisonFilter, + CompoundFilter, + FilterTranslator, + FilterType, + filters_contain_numeric_comparison, +) from feast.infra.infra_object import InfraObject from feast.infra.key_encoding_utils import ( deserialize_entity_key, @@ -96,6 +104,95 @@ if milvus_type: FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING[feast_type] = milvus_type +logger = logging.getLogger(__name__) + +MILVUS_NATIVE_NUMERIC_TYPES = { + DataType.INT32, + DataType.INT64, + DataType.FLOAT, + DataType.DOUBLE, + DataType.BOOL, +} + + +def _milvus_escape_string(s: str) -> str: + """Escape a string for safe use inside a Milvus single-quoted literal. + + Backslashes must be escaped first; otherwise a trailing backslash in the + input would combine with the escaped quote to break out of the literal. + Also escapes quotes and common control characters that could disrupt + Milvus boolean expressions. + """ + return ( + s.replace("\\", "\\\\") + .replace("'", "\\'") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + ) + + +def _milvus_fmt(value: Any) -> str: + """Format a Python value for use in a Milvus boolean expression. + + Handles numeric types natively (unquoted) so that Milvus performs + numeric comparison instead of lexicographic string comparison. + Bool is checked first because Python ``bool`` is a subclass of ``int``. + """ + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + return f"'{_milvus_escape_string(str(value))}'" + + +class MilvusFilterTranslator(FilterTranslator): + """Translates Feast filters into Milvus boolean expression strings.""" + + def translate(self, filters: FilterType) -> Optional[str]: + if filters is None: + return None + return self._dispatch(filters) + + def translate_comparison(self, f: ComparisonFilter) -> str: + key, value, op_type = f.key, f.value, f.type + + if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", key): + raise ValueError( + f"Invalid filter key: {key!r}. Keys must be valid field identifiers." + ) + + milvus_ops = {"gt": ">", "gte": ">=", "lt": "<", "lte": "<="} + + if op_type == "eq": + return f"{key} == {_milvus_fmt(value)}" + elif op_type == "ne": + return f"{key} != {_milvus_fmt(value)}" + elif op_type in milvus_ops: + return f"{key} {milvus_ops[op_type]} {_milvus_fmt(value)}" + elif op_type in ("in", "nin"): + if not isinstance(value, list): + raise ValueError( + f"'{op_type}' filter requires a list value, got {type(value)}" + ) + formatted = [_milvus_fmt(v) for v in value] + kw = "not in" if op_type == "nin" else "in" + return f"{key} {kw} [{', '.join(formatted)}]" + raise ValueError(f"Unsupported comparison operator: {op_type}") + + def translate_compound(self, f: CompoundFilter) -> str: + if not f.filters: + return "" + clauses = [] + for sub_filter in f.filters: + clause = self._dispatch(sub_filter) + if clause: + clauses.append(f"({clause})") + if not clauses: + return "" + operator = " and " if f.type == "and" else " or " + return operator.join(clauses) + class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): """ @@ -115,6 +212,7 @@ class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): nlist: Optional[int] = 128 username: Optional[StrictStr] = "" password: Optional[StrictStr] = "" + enable_openai_compatible_store: Optional[bool] = False varchar_max_length: Optional[int] = 65535 @field_validator("varchar_max_length") @@ -199,6 +297,7 @@ def _get_or_create_collection( "created_timestamp", ] fields_to_add = [f for f in table.schema if f.name not in fields_to_exclude] + use_typed = config.online_store.enable_openai_compatible_store for field in fields_to_add: dtype = FEAST_PRIMITIVE_TO_MILVUS_TYPE_MAPPING.get(field.dtype) if dtype is None and isinstance(field.dtype, ComplexFeastType): @@ -212,6 +311,13 @@ def _get_or_create_collection( dim=config.online_store.embedding_dim, ) ) + elif use_typed and dtype in MILVUS_NATIVE_NUMERIC_TYPES: + fields.append( + FieldSchema( + name=field.name, + dtype=dtype, + ) + ) else: if "max_length" in field.tags: try: @@ -314,14 +420,19 @@ def online_write_batch( entity_batch_to_insert = [] unique_entities: dict[str, dict[str, Any]] = {} required_fields = {field["name"] for field in collection["fields"]} + collection_field_types = { + field["name"]: field["type"] for field in collection["fields"] + } + schema_internal_fields = {"event_ts", "created_ts"} + collection_has_native_numerics = any( + collection_field_types.get(name) in MILVUS_NATIVE_NUMERIC_TYPES + for name in required_fields - schema_internal_fields + ) for entity_key, values_dict, timestamp, created_ts in data: - # need to construct the composite primary key also need to handle the fact that entities are a list entity_key_str = serialize_entity_key( entity_key, entity_key_serialization_version=config.entity_key_serialization_version, ).hex() - # to recover the entity key just run: - # deserialize_entity_key(bytes.fromhex(entity_key_str), entity_key_serialization_version=3) composite_key_name = _get_composite_key_name(table) timestamp_int = int(to_naive_utc(timestamp).timestamp() * 1e6) @@ -339,6 +450,7 @@ def online_write_batch( values_dict, vector_cols=vector_cols, serialize_to_string=True, + use_native_numeric_types=collection_has_native_numerics, ) # Remove timestamp fields that are handled separately to avoid conflicts @@ -357,13 +469,15 @@ def online_write_batch( "created_ts": created_ts_int, } single_entity_record.update(values_dict) - # Ensure all required fields exist, setting missing ones to defaults for field in required_fields: if field not in single_entity_record: + field_type = collection_field_types.get(field, DataType.VARCHAR) if field == "_placeholder_vector": single_entity_record[field] = [float("nan")] else: - single_entity_record[field] = "" + single_entity_record[field] = _default_for_milvus_type( + field_type + ) # Store only the latest event timestamp per entity if ( entity_key_str not in unique_entities @@ -569,6 +683,7 @@ def retrieve_online_documents_v2( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -591,7 +706,9 @@ def retrieve_online_documents_v2( List of tuples containing the event timestamp, entity key, and feature values """ entity_name_feast_primitive_type_map = { - k.name: k.dtype for k in table.entity_columns + k.name: k.dtype + for k in list(table.entity_columns) + list(table.features) + if isinstance(k.dtype, PrimitiveFeastType) } self.client = self._connect(config) collection_name = _table_id( @@ -631,6 +748,38 @@ def retrieve_online_documents_v2( self.client.load_collection(collection_name) + if filters and filters_contain_numeric_comparison(filters): + collection_field_types = { + f["name"]: f["type"] for f in collection["fields"] + } + schema_internal_fields = {"event_ts", "created_ts"} + has_native = any( + collection_field_types.get(name) in MILVUS_NATIVE_NUMERIC_TYPES + for name in collection_field_types + if name not in schema_internal_fields + ) + if not has_native: + logger.warning( + "Numeric comparison filters (gt, gte, lt, lte) are being used " + "but this collection stores numeric fields as VARCHAR. This " + "causes lexicographic comparison instead of numeric comparison " + "(e.g. '9' > '100' is True as a string). To fix this, set " + "'enable_openai_compatible_store: true' in your online_store " + "config, then teardown and re-apply your feature store to " + "recreate collections with native numeric types." + ) + + metadata_filter_expr = MilvusFilterTranslator().translate(filters) + + def _combine_exprs(*parts: Optional[str]) -> Optional[str]: + """Combine non-empty Milvus boolean expressions with AND.""" + active = [p for p in parts if p] + if not active: + return None + if len(active) == 1: + return active[0] + return " and ".join(f"({p})" for p in active) + if ( embedding is not None and query_string is not None @@ -648,22 +797,20 @@ def retrieve_online_documents_v2( "No string fields found in the feature view for text search in hybrid mode" ) - # Create a filter expression for text search + escaped_query = _milvus_escape_string(query_string) filter_expressions = [] for field in string_field_list: if field in output_fields: - filter_expressions.append(f"{field} LIKE '%{query_string}%'") + filter_expressions.append(f"{field} LIKE '%{escaped_query}%'") - # Combine filter expressions with OR - filter_expr = " OR ".join(filter_expressions) if filter_expressions else "" + text_filter = " OR ".join(filter_expressions) if filter_expressions else "" + combined_filter = _combine_exprs(text_filter, metadata_filter_expr) - # Vector search with text filter search_params = { "metric_type": distance_metric or config.online_store.metric_type, "params": {"nprobe": 10}, } - # For hybrid search, use filter parameter instead of expr results = self.client.search( collection_name=collection_name, data=[embedding], @@ -671,7 +818,7 @@ def retrieve_online_documents_v2( search_params=search_params, limit=top_k, output_fields=output_fields, - filter=filter_expr if filter_expr else None, + filter=combined_filter, ) elif embedding is not None and config.online_store.vector_enabled: @@ -688,6 +835,7 @@ def retrieve_online_documents_v2( search_params=search_params, limit=top_k, output_fields=output_fields, + filter=metadata_filter_expr, ) elif query_string is not None: @@ -703,21 +851,24 @@ def retrieve_online_documents_v2( "No string fields found in the feature view for text search" ) + escaped_query = _milvus_escape_string(query_string) filter_expressions = [] for field in string_field_list: if field in output_fields: - filter_expressions.append(f"{field} LIKE '%{query_string}%'") + filter_expressions.append(f"{field} LIKE '%{escaped_query}%'") - filter_expr = " OR ".join(filter_expressions) + text_filter = " OR ".join(filter_expressions) - if not filter_expr: + if not text_filter: raise ValueError( "No text fields found in requested features for search" ) + combined_filter = _combine_exprs(text_filter, metadata_filter_expr) + query_results = self.client.query( collection_name=collection_name, - filter=filter_expr, + filter=combined_filter or text_filter, output_fields=output_fields, limit=top_k, ) @@ -778,17 +929,37 @@ def retrieve_online_documents_v2( PrimitiveFeastType.INT32, ]: res[field] = ValueProto(int64_val=int(field_value)) + elif entity_name_feast_primitive_type_map.get( + field, PrimitiveFeastType.INVALID + ) in [ + PrimitiveFeastType.FLOAT64, + PrimitiveFeastType.FLOAT32, + ]: + res[field] = ValueProto(double_val=float(field_value)) + elif ( + entity_name_feast_primitive_type_map.get( + field, PrimitiveFeastType.INVALID + ) + == PrimitiveFeastType.BOOL + ): + res[field] = ValueProto(bool_val=bool(field_value)) elif field == composite_key_name: pass elif isinstance(field_value, bytes): val.ParseFromString(field_value) res[field] = val + elif isinstance(field_value, int): + res[field] = ValueProto(int64_val=field_value) + elif isinstance(field_value, float): + res[field] = ValueProto(double_val=field_value) else: - val.string_val = field_value + val.string_val = str(field_value) res[field] = val - distance = hit.get("distance", None) + raw_distance = hit.get("distance", None) res["distance"] = ( - ValueProto(float_val=distance) if distance else ValueProto() + ValueProto(float_val=raw_distance) + if raw_distance is not None + else ValueProto() ) result_list.append((res_ts, entity_key_proto, res if res else None)) return result_list @@ -821,10 +992,25 @@ def _get_composite_key_name(table: FeatureView) -> str: return "_".join([field.name for field in table.entity_columns]) + "_pk" +_MILVUS_TYPE_DEFAULTS: Dict[DataType, Any] = { + DataType.INT32: 0, + DataType.INT64: 0, + DataType.FLOAT: 0.0, + DataType.DOUBLE: 0.0, + DataType.BOOL: False, + DataType.VARCHAR: "", +} + + +def _default_for_milvus_type(dtype: DataType) -> Any: + return _MILVUS_TYPE_DEFAULTS.get(dtype, "") + + def _extract_proto_values_to_dict( input_dict: Dict[str, Any], vector_cols: List[str], - serialize_to_string=False, + serialize_to_string: bool = False, + use_native_numeric_types: bool = False, ) -> Dict[str, Any]: numeric_vector_list_types = [ k @@ -857,13 +1043,16 @@ def _extract_proto_values_to_dict( not in ["string_val", "bytes_val", "unix_timestamp_val"] + numeric_types ): - # For complex types, use base64 encoding instead of decode vector_values = base64.b64encode( feature_values.SerializeToString() ).decode("utf-8") elif proto_val_type == "bytes_val": byte_data = getattr(feature_values, proto_val_type) vector_values = base64.b64encode(byte_data).decode("utf-8") + elif ( + use_native_numeric_types and proto_val_type in numeric_types + ): + vector_values = getattr(feature_values, proto_val_type) else: if not isinstance(feature_values, str): vector_values = str( @@ -874,8 +1063,13 @@ def _extract_proto_values_to_dict( output_dict[feature_name] = vector_values else: if serialize_to_string: - if not isinstance(feature_values, str): - feature_values = str(feature_values) - output_dict[feature_name] = feature_values + if use_native_numeric_types and isinstance( + feature_values, (int, float) + ): + output_dict[feature_name] = feature_values + else: + if not isinstance(feature_values, str): + feature_values = str(feature_values) + output_dict[feature_name] = feature_values return output_dict diff --git a/sdk/python/feast/infra/online_stores/mongodb_online_store/mongodb.py b/sdk/python/feast/infra/online_stores/mongodb_online_store/mongodb.py index d0105607d43..16dd572bf5b 100644 --- a/sdk/python/feast/infra/online_stores/mongodb_online_store/mongodb.py +++ b/sdk/python/feast/infra/online_stores/mongodb_online_store/mongodb.py @@ -20,6 +20,12 @@ from feast.batch_feature_view import BatchFeatureView from feast.entity import Entity from feast.feature_view import FeatureView +from feast.filter_models import ( + ComparisonFilter, + CompoundFilter, + FilterTranslator, + FilterType, +) from feast.infra.key_encoding_utils import deserialize_entity_key, serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore from feast.infra.online_stores.vector_store import VectorStoreConfig @@ -37,6 +43,64 @@ DRIVER_METADATA = DriverInfo(name="Feast", version=feast.version.get_version()) +_MONGO_COMPARISON_OPS: Dict[str, str] = { + "eq": "$eq", + "ne": "$ne", + "gt": "$gt", + "gte": "$gte", + "lt": "$lt", + "lte": "$lte", +} + + +class MongoDBFilterTranslator(FilterTranslator): + """Translates Feast filters into MongoDB Query Language (MQL) expressions. + + Filter keys are prefixed with the feature path so they match the + document layout used by :class:`MongoDBOnlineStore`:: + + features.. + """ + + def __init__(self, table_name: str): + self.table_name = table_name + + def _field(self, key: str) -> str: + return f"features.{self.table_name}.{key}" + + def translate(self, filters: FilterType) -> Optional[Dict[str, Any]]: + if filters is None: + return None + return self._dispatch(filters) + + def translate_comparison(self, f: ComparisonFilter) -> Dict[str, Any]: + field = self._field(f.key) + + if f.type in _MONGO_COMPARISON_OPS: + return {field: {_MONGO_COMPARISON_OPS[f.type]: f.value}} + + if f.type == "in": + if not isinstance(f.value, list): + raise ValueError( + f"'in' filter requires a list value, got {type(f.value)}" + ) + return {field: {"$in": f.value}} + + if f.type == "nin": + if not isinstance(f.value, list): + raise ValueError( + f"'nin' filter requires a list value, got {type(f.value)}" + ) + return {field: {"$nin": f.value}} + + raise ValueError(f"Unsupported comparison operator: {f.type}") + + def translate_compound(self, f: CompoundFilter) -> Dict[str, Any]: + clauses = [self._dispatch(sub) for sub in f.filters] + if f.type == "and": + return {"$and": clauses} + return {"$or": clauses} + class MongoDBOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): """MongoDB configuration. @@ -232,6 +296,7 @@ def retrieve_online_documents_v2( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -277,16 +342,20 @@ def retrieve_online_documents_v2( query_vector = [float(v) for v in embedding] num_candidates = max(top_k * 10, 100) + vector_search_stage: Dict[str, Any] = { + "index": idx_name, + "path": path, + "queryVector": query_vector, + "numCandidates": num_candidates, + "limit": top_k, + } + + mql_filter = MongoDBFilterTranslator(table.name).translate(filters) + if mql_filter: + vector_search_stage["filter"] = mql_filter + pipeline: List[Dict[str, Any]] = [ - { - "$vectorSearch": { - "index": idx_name, - "path": path, - "queryVector": query_vector, - "numCandidates": num_candidates, - "limit": top_k, - } - }, + {"$vectorSearch": vector_search_stage}, { "$addFields": { "score": {"$meta": "vectorSearchScore"}, diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 87feba3f830..cdf06639fe0 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -25,6 +25,7 @@ from feast.errors import VersionedOnlineReadNotSupported from feast.feature_service import FeatureService from feast.feature_view import FeatureView +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.infra.infra_object import InfraObject from feast.infra.registry.base_registry import BaseRegistry from feast.infra.supported_async_methods import SupportedAsyncMethods @@ -927,6 +928,7 @@ def retrieve_online_documents_v2( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -946,6 +948,8 @@ def retrieve_online_documents_v2( embedding: The embeddings to use for retrieval (optional) top_k: The number of documents to retrieve. query_string: The query string to search for using keyword search (bm25) (optional) + filters: Optional metadata filters (ComparisonFilter or CompoundFilter) + to narrow results before ranking. Returns: object: A list of top k closest documents to the specified embedding. Each item in the list is a tuple diff --git a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py index 4a79349e91c..09be88c991d 100644 --- a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py +++ b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py @@ -21,8 +21,19 @@ from psycopg_pool import AsyncConnectionPool, ConnectionPool from feast import Entity, FeatureView, ValueType +from feast.filter_models import ( + ComparisonFilter, + CompoundFilter, + FilterTranslator, + FilterType, + filters_contain_numeric_comparison, +) from feast.infra.key_encoding_utils import get_list_val_str, serialize_entity_key -from feast.infra.online_stores.helpers import _to_naive_utc, compute_table_id +from feast.infra.online_stores.helpers import ( + _to_naive_utc, + compute_table_id, + extract_text_and_num, +) from feast.infra.online_stores.online_store import OnlineStore from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.infra.utils.postgres.connection_utils import ( @@ -44,9 +55,110 @@ "inner_product": "<#>", } +_PG_COMPARISON_OPS: Dict[str, str] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +class PostgresFilterTranslator(FilterTranslator): + """Translates Feast filters into Postgres SQL WHERE clause fragments.""" + + def __init__(self, table_name: str, alias: Optional[str] = None): + self.table_name = table_name + self.alias = alias + + def translate(self, filters: FilterType) -> Tuple[sql.Composable, List[Any]]: + if filters is None: + return sql.SQL(""), [] + return self._dispatch(filters) + + def translate_comparison( + self, f: ComparisonFilter + ) -> Tuple[sql.Composable, List[Any]]: + key, value, op_type = f.key, f.value, f.type + ek_col = f"{self.alias}.entity_key" if self.alias else "entity_key" + + if op_type in _PG_COMPARISON_OPS: + col, db_value = _pg_filter_col_and_val(value) + clause = sql.SQL( + "{ek_col} IN (SELECT entity_key FROM {tbl} WHERE feature_name = %s AND {col} {op} %s)" + ).format( + ek_col=sql.SQL(ek_col), + tbl=sql.Identifier(self.table_name), + col=sql.Identifier(col), + op=sql.SQL(_PG_COMPARISON_OPS[op_type]), + ) + return clause, [key, db_value] + + if op_type == "in": + if not isinstance(value, list): + raise ValueError( + f"'in' filter requires a list value, got {type(value)}" + ) + placeholders = sql.SQL(", ").join([sql.Placeholder()] * len(value)) + col, _ = _pg_filter_col_and_val(value[0]) if value else ("value_text", None) + db_values = [_pg_filter_col_and_val(v)[1] for v in value] + clause = sql.SQL( + "{ek_col} IN (SELECT entity_key FROM {tbl} WHERE feature_name = %s AND {col} IN ({phs}))" + ).format( + ek_col=sql.SQL(ek_col), + tbl=sql.Identifier(self.table_name), + col=sql.Identifier(col), + phs=placeholders, + ) + return clause, [key] + db_values + + if op_type == "nin": + if not isinstance(value, list): + raise ValueError( + f"'nin' filter requires a list value, got {type(value)}" + ) + placeholders = sql.SQL(", ").join([sql.Placeholder()] * len(value)) + col, _ = _pg_filter_col_and_val(value[0]) if value else ("value_text", None) + db_values = [_pg_filter_col_and_val(v)[1] for v in value] + clause = sql.SQL( + "{ek_col} IN (SELECT entity_key FROM {tbl} WHERE feature_name = %s AND {col} NOT IN ({phs}))" + ).format( + ek_col=sql.SQL(ek_col), + tbl=sql.Identifier(self.table_name), + col=sql.Identifier(col), + phs=placeholders, + ) + return clause, [key] + db_values + + raise ValueError(f"Unknown comparison operator: {op_type}") + + def translate_compound(self, f: CompoundFilter) -> Tuple[sql.Composable, List[Any]]: + if not f.filters: + return sql.SQL(""), [] + parts: List[sql.Composable] = [] + all_params: List[Any] = [] + for sub in f.filters: + sub_clause, sub_params = self._dispatch(sub) + parts.append(sub_clause) + all_params.extend(sub_params) + joiner = sql.SQL(" AND " if f.type == "and" else " OR ") + combined = sql.SQL("(") + joiner.join(parts) + sql.SQL(")") + return combined, all_params + + +def _pg_filter_col_and_val(value: Any) -> Tuple[str, Any]: + """Return the appropriate column name and DB-ready value for a filter value.""" + if isinstance(value, bool): + return "value_num", 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return "value_num", float(value) + return "value_text", str(value) + class PostgreSQLOnlineStoreConfig(PostgreSQLConfig, VectorStoreConfig): type: Literal["postgres"] = "postgres" + enable_openai_compatible_store: Optional[bool] = False class PostgreSQLOnlineStore(OnlineStore): @@ -55,6 +167,7 @@ class PostgreSQLOnlineStore(OnlineStore): _conn_async: Optional[AsyncConnection] = None _conn_pool_async: Optional[AsyncConnectionPool] = None + _table_has_value_num: Optional[Dict[str, bool]] = None @contextlib.contextmanager def _get_conn( @@ -96,6 +209,33 @@ async def _get_conn_async( await self._conn_async.set_autocommit(autocommit) yield self._conn_async + def _check_table_has_value_num( + self, cur, table_name: str, config: RepoConfig + ) -> bool: + """Check if the value_num column exists in the given table, with caching.""" + if self._table_has_value_num is None: + self._table_has_value_num = {} + if table_name in self._table_has_value_num: + return self._table_has_value_num[table_name] + + schema_name = config.online_store.db_schema or config.online_store.user + cur.execute( + """ + SELECT 1 FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s AND column_name = 'value_num' + """, + (schema_name, table_name), + ) + exists = cur.fetchone() is not None + self._table_has_value_num[table_name] = exists + return exists + + @staticmethod + def _filters_need_value_num( + filters: Union[ComparisonFilter, CompoundFilter], + ) -> bool: + return filters_contain_numeric_comparison(filters) + def online_write_batch( self, config: RepoConfig, @@ -105,65 +245,71 @@ def online_write_batch( ], progress: Optional[Callable[[int], Any]], ) -> None: - # Format insert values - insert_values = [] - for entity_key, values, timestamp, created_ts in data: - entity_key_bin = serialize_entity_key( - entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, + table_name = _table_id( + config.project, table, config.registry.enable_online_feature_view_versioning + ) + + with self._get_conn(config) as conn, conn.cursor() as cur: + enable_value_num = getattr( + config.online_store, "enable_openai_compatible_store", False ) - timestamp = _to_naive_utc(timestamp) - if created_ts is not None: - created_ts = _to_naive_utc(created_ts) + has_value_num_col = self._check_table_has_value_num(cur, table_name, config) + compute_value_num = has_value_num_col + if enable_value_num and not has_value_num_col: + logging.warning( + "enable_openai_compatible_store is True but value_num column " + "not found in table '%s'. Run `feast apply` to add it. " + "Writing without value_num.", + table_name, + ) + compute_value_num = False + + columns = ["entity_key", "feature_name", "value", "value_text"] + if has_value_num_col: + columns.append("value_num") + columns.extend(["vector_value", "event_ts", "created_ts"]) + + pk_cols = {"entity_key", "feature_name"} + col_ids = sql.SQL(", ").join(sql.Identifier(c) for c in columns) + placeholders = sql.SQL(", ").join([sql.Placeholder()] * len(columns)) + update_set = sql.SQL(", ").join( + sql.SQL("{} = EXCLUDED.{}").format(sql.Identifier(c), sql.Identifier(c)) + for c in columns + if c not in pk_cols + ) + sql_query = sql.SQL( + "INSERT INTO {} ({}) VALUES ({}) " + "ON CONFLICT (entity_key, feature_name) DO UPDATE SET {};" + ).format(sql.Identifier(table_name), col_ids, placeholders, update_set) + + insert_values: List[Tuple[Any, ...]] = [] + for entity_key, values, timestamp, created_ts in data: + entity_key_bin = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + timestamp = _to_naive_utc(timestamp) + if created_ts is not None: + created_ts = _to_naive_utc(created_ts) - for feature_name, val in values.items(): - vector_val = None - value_text = None + for feature_name, val in values.items(): + value_text, value_num = extract_text_and_num(val, compute_value_num) - # Check if the feature type is STRING - if val.WhichOneof("val") == "string_val": - value_text = val.string_val + vector_val = None + if config.online_store.vector_enabled: + vector_val = get_list_val_str(val) - if config.online_store.vector_enabled: - vector_val = get_list_val_str(val) - insert_values.append( - ( + row: List[Any] = [ entity_key_bin, feature_name, val.SerializeToString(), value_text, - vector_val, - timestamp, - created_ts, - ) - ) + ] + if has_value_num_col: + row.append(value_num) + row.extend([vector_val, timestamp, created_ts]) + insert_values.append(tuple(row)) - # Create insert query - sql_query = sql.SQL( - """ - INSERT INTO {} - (entity_key, feature_name, value, value_text, vector_value, event_ts, created_ts) - VALUES (%s, %s, %s, %s, %s, %s, %s) - ON CONFLICT (entity_key, feature_name) DO - UPDATE SET - value = EXCLUDED.value, - value_text = EXCLUDED.value_text, - vector_value = EXCLUDED.vector_value, - event_ts = EXCLUDED.event_ts, - created_ts = EXCLUDED.created_ts; - """ - ).format( - sql.Identifier( - _table_id( - config.project, - table, - config.registry.enable_online_feature_view_versioning, - ) - ) - ) - - # Push data into the online store - with self._get_conn(config) as conn, conn.cursor() as cur: cur.executemany(sql_query, insert_values) conn.commit() @@ -344,6 +490,14 @@ def update( f.dtype.to_value_type() == ValueType.STRING for f in table.features ) + value_num_col = ( + sql.SQL("value_num DOUBLE PRECISION NULL,") + if getattr( + config.online_store, "enable_openai_compatible_store", False + ) + else sql.SQL("") + ) + cur.execute( sql.SQL( """ @@ -352,7 +506,8 @@ def update( entity_key BYTEA, feature_name TEXT, value BYTEA, - value_text TEXT NULL, -- Added for FTS + value_text TEXT NULL, + {} vector_value {} NULL, event_ts TIMESTAMPTZ, created_ts TIMESTAMPTZ, @@ -362,12 +517,25 @@ def update( """ ).format( sql.Identifier(table_name), + value_num_col, sql.SQL(vector_value_type), sql.Identifier(f"{table_name}_ek"), sql.Identifier(table_name), ) ) + if getattr( + config.online_store, "enable_openai_compatible_store", False + ): + cur.execute( + sql.SQL( + """ALTER TABLE {} ADD COLUMN IF NOT EXISTS value_num DOUBLE PRECISION NULL;""" + ).format(sql.Identifier(table_name)) + ) + if self._table_has_value_num is None: + self._table_has_value_num = {} + self._table_has_value_num[table_name] = True + if has_string_features: cur.execute( sql.SQL( @@ -519,6 +687,7 @@ def retrieve_online_documents_v2( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -548,6 +717,16 @@ def retrieve_online_documents_v2( if embedding is None and query_string is None: raise ValueError("Either embedding or query_string must be provided") + if filters is not None: + if not getattr( + config.online_store, "enable_openai_compatible_store", False + ): + raise ValueError( + "Metadata filtering requires `enable_openai_compatible_store: true` " + "in your online store config. After setting it, run `feast apply` " + "to update the database schema." + ) + distance_metric = distance_metric or "L2" if distance_metric not in SUPPORTED_DISTANCE_METRICS_DICT: @@ -571,12 +750,36 @@ def retrieve_online_documents_v2( ) with self._get_conn(config, autocommit=True) as conn, conn.cursor() as cur: + if filters is not None and self._filters_need_value_num(filters): + if not self._check_table_has_value_num(cur, table_name, config): + raise ValueError( + "Numerical filtering requires the `value_num` column. " + "Run `feast apply` to add it." + ) + query = None params: Any = None + filter_clause, filter_params = PostgresFilterTranslator( + table_name, alias="t1" + ).translate(filters) + has_filters = bool(filter_params) or ( + filters is not None + and not filter_params + and filter_clause.as_string(conn) != "" + ) + if embedding is not None and query_string is not None and string_fields: # Case 1: Hybrid Search (vector + text) tsquery_str = " & ".join(query_string.split()) + + outer_where_parts: list[sql.Composable] = [ + sql.SQL("t1.feature_name = ANY(%s)") + ] + if has_filters: + outer_where_parts.append(filter_clause) + outer_where = sql.SQL(" AND ").join(outer_where_parts) + query = sql.SQL( """ WITH vector_candidates AS ( @@ -636,15 +839,16 @@ def retrieve_online_documents_v2( t1.created_ts FROM {table_name} t1 INNER JOIN scored s ON t1.entity_key = s.entity_key - WHERE t1.feature_name = ANY(%s) + WHERE {outer_where} ORDER BY s.text_rank DESC, s.distance """ ).format( distance_metric_sql=sql.SQL(distance_metric_sql), table_name=sql.Identifier(table_name), + outer_where=outer_where, top_k=sql.Literal(top_k), ) - params = ( + base_params: list[Any] = [ embedding, tsquery_str, string_fields, @@ -653,9 +857,18 @@ def retrieve_online_documents_v2( tsquery_str, string_fields, requested_features, - ) + ] + if has_filters: + base_params.extend(filter_params) + params = tuple(base_params) + elif embedding is not None: # Case 2: Vector Search Only + outer_where_parts = [sql.SQL("t1.feature_name = ANY(%s)")] + if has_filters: + outer_where_parts.append(filter_clause) + outer_where = sql.SQL(" AND ").join(outer_where_parts) + query = sql.SQL( """ WITH vector_matches AS ( @@ -678,25 +891,36 @@ def retrieve_online_documents_v2( t1.created_ts FROM {table_name} t1 INNER JOIN vector_matches t2 ON t1.entity_key = t2.entity_key - WHERE t1.feature_name = ANY(%s) + WHERE {outer_where} ORDER BY t2.distance """ ).format( distance_metric_sql=sql.SQL(distance_metric_sql), table_name=sql.Identifier(table_name), + outer_where=outer_where, top_k=sql.Literal(top_k), ) - params = (embedding, requested_features) + base_params = [embedding, requested_features] + if has_filters: + base_params.extend(filter_params) + params = tuple(base_params) elif query_string is not None and string_fields: # Case 3: Text Search Only tsquery_str = " & ".join(query_string.split()) + + outer_where_parts = [sql.SQL("t1.feature_name = ANY(%s)")] + if has_filters: + outer_where_parts.append(filter_clause) + outer_where = sql.SQL(" AND ").join(outer_where_parts) + query = sql.SQL( """ WITH text_matches AS ( SELECT DISTINCT entity_key, ts_rank(to_tsvector('english', value_text), to_tsquery('english', %s)) as text_rank FROM {table_name} - WHERE feature_name = ANY(%s) AND to_tsvector('english', value_text) @@ to_tsquery('english', %s) + WHERE feature_name = ANY(%s) + AND to_tsvector('english', value_text) @@ to_tsquery('english', %s) ORDER BY text_rank DESC LIMIT {top_k} ) @@ -711,14 +935,23 @@ def retrieve_online_documents_v2( t1.created_ts FROM {table_name} t1 INNER JOIN text_matches t2 ON t1.entity_key = t2.entity_key - WHERE t1.feature_name = ANY(%s) + WHERE {outer_where} ORDER BY t2.text_rank DESC """ ).format( table_name=sql.Identifier(table_name), + outer_where=outer_where, top_k=sql.Literal(top_k), ) - params = (tsquery_str, string_fields, tsquery_str, requested_features) + base_params = [ + tsquery_str, + string_fields, + tsquery_str, + requested_features, + ] + if has_filters: + base_params.extend(filter_params) + params = tuple(base_params) else: raise ValueError( diff --git a/sdk/python/feast/infra/online_stores/remote.py b/sdk/python/feast/infra/online_stores/remote.py index 05a6b05dbea..7a60f091689 100644 --- a/sdk/python/feast/infra/online_stores/remote.py +++ b/sdk/python/feast/infra/online_stores/remote.py @@ -34,6 +34,7 @@ from feast import Entity, FeatureView, RepoConfig from feast.feature_service import FeatureService +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.infra.online_stores.helpers import _to_naive_utc from feast.infra.online_stores.online_store import OnlineStore from feast.infra.registry.base_registry import BaseRegistry @@ -438,6 +439,7 @@ def retrieve_online_documents_v2( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -456,6 +458,7 @@ def retrieve_online_documents_v2( top_k, distance_metric, query_string, + filters=filters, api_version=2, ) response = get_remote_online_documents(config=config, req_body=req_body) @@ -464,20 +467,31 @@ def retrieve_online_documents_v2( response_json = json.loads(response.text) event_ts: Optional[datetime] = self._get_event_ts(response_json) + metadata = response_json.get("metadata") or {} + feature_names = metadata.get("feature_names") or [] + results = response_json.get("results") or [] + + if not feature_names or not results: + logger.debug( + "Empty metadata or results in retrieve_online_documents_v2 response." + ) + return [] + # Create feature name to index mapping for efficient lookup feature_name_to_index = { - name: idx - for idx, name in enumerate(response_json["metadata"]["feature_names"]) + name: idx for idx, name in enumerate(feature_names) } # Process each result row - num_results = ( - len(response_json["results"][0]["values"]) - if response_json["results"] - else 0 - ) + first_result_values = results[0].get("values") or [] + num_results = len(first_result_values) result_tuples = [] - + if requested_features is None: + requested_features = [] + else: + requested_features = list(requested_features) + if "distance" not in requested_features: + requested_features.append("distance") for row_idx in range(num_results): # Build feature values dictionary for requested features feature_values_dict = {} @@ -639,6 +653,7 @@ def _construct_online_documents_v2_api_json_request( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, api_version: Optional[int] = 2, ) -> dict: api_requested_features = [] @@ -646,7 +661,7 @@ def _construct_online_documents_v2_api_json_request( for requested_feature in requested_features: api_requested_features.append(f"{table.name}:{requested_feature}") - return { + body: Dict[str, Any] = { "features": api_requested_features, "query": embedding, "top_k": top_k, @@ -654,12 +669,19 @@ def _construct_online_documents_v2_api_json_request( "query_string": query_string, "api_version": api_version, } - - def _get_event_ts(self, response_json) -> datetime: - event_ts = "" - if len(response_json["results"]) > 1: - event_ts = response_json["results"][1]["event_timestamps"][0] - return datetime.fromisoformat(event_ts.replace("Z", "+00:00")) + if filters is not None: + body["filters"] = filters.model_dump() + return body + + def _get_event_ts(self, response_json) -> Optional[datetime]: + results = response_json.get("results") or [] + if len(results) > 1: + event_timestamps = results[1].get("event_timestamps") or [] + if event_timestamps: + return datetime.fromisoformat( + event_timestamps[0].replace("Z", "+00:00") + ) + return None def _construct_entity_key_from_response( self, @@ -757,16 +779,20 @@ def get_remote_online_features( def get_remote_online_documents( session: requests.Session, config: RepoConfig, req_body: dict ) -> requests.Response: - if config.online_store.cert: - return session.post( - f"{config.online_store.path}/retrieve-online-documents", - json=req_body, - verify=config.online_store.cert, - ) - else: - return session.post( - f"{config.online_store.path}/retrieve-online-documents", json=req_body - ) + search_paths = ("/search", "/retrieve-online-documents") + last_response: Optional[requests.Response] = None + for path in search_paths: + url = f"{config.online_store.path}{path}" + if config.online_store.cert: + last_response = session.post( + url, json=req_body, verify=config.online_store.cert + ) + else: + last_response = session.post(url, json=req_body) + if last_response.status_code != 404: + return last_response + assert last_response is not None + return last_response @rest_error_handling_decorator diff --git a/sdk/python/feast/infra/online_stores/scylladb_online_store/scylladb.py b/sdk/python/feast/infra/online_stores/scylladb_online_store/scylladb.py index 51a22e07a76..f68d8e5aacf 100644 --- a/sdk/python/feast/infra/online_stores/scylladb_online_store/scylladb.py +++ b/sdk/python/feast/infra/online_stores/scylladb_online_store/scylladb.py @@ -11,6 +11,7 @@ Optional, Sequence, Tuple, + Union, ) from cassandra.auth import PlainTextAuthProvider @@ -26,6 +27,7 @@ from pydantic import StrictFloat, StrictInt, StrictStr from feast import Entity, FeatureView, RepoConfig +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.infra.key_encoding_utils import deserialize_entity_key, serialize_entity_key from feast.infra.online_stores.online_store import OnlineStore from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto @@ -651,6 +653,7 @@ def retrieve_online_documents_v2( top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -678,12 +681,17 @@ def retrieve_online_documents_v2( store-level ``vector_similarity_function`` config value or the per-feature tag. query_string: Unused (reserved for future hybrid text+vector search). + filters: Unused (metadata filtering not yet supported for ScyllaDB). include_feature_view_version_metadata: Unused. Returns: List of ``(event_timestamp, entity_key_proto, feature_values)`` tuples ordered from most to least similar. """ + if filters is not None: + raise NotImplementedError( + "Metadata filtering is not supported by the ScyllaDB online store." + ) if embedding is None: raise ValueError( "retrieve_online_documents_v2 requires a non-None 'embedding' " diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index 74ddd030d57..d56c430a996 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -37,13 +37,20 @@ from feast import Entity from feast.feature_view import FeatureView from feast.field import Field +from feast.filter_models import ( + ComparisonFilter, + CompoundFilter, + FilterTranslator, + FilterType, + filters_contain_numeric_comparison, +) from feast.infra.infra_object import SQLITE_INFRA_OBJECT_CLASS_TYPE, InfraObject from feast.infra.key_encoding_utils import ( deserialize_entity_key, serialize_entity_key, serialize_f32, ) -from feast.infra.online_stores.helpers import compute_table_id +from feast.infra.online_stores.helpers import compute_table_id, extract_text_and_num from feast.infra.online_stores.online_store import OnlineStore from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.labeling.label_view import LabelView @@ -103,6 +110,97 @@ def convert_timestamp(val: bytes): sqlite3.register_converter("timestamp", convert_timestamp) +_SQLITE_COMPARISON_OPS: Dict[str, str] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +class SqliteFilterTranslator(FilterTranslator): + """Translates Feast filters into SQLite WHERE clause fragments.""" + + def __init__(self, table_name: str, alias: Optional[str] = None): + self.table_name = table_name + self.alias = alias + + def translate(self, filters: FilterType) -> Tuple[str, List[Any]]: + if filters is None: + return "", [] + return self._dispatch(filters) + + def translate_comparison(self, f: ComparisonFilter) -> Tuple[str, List[Any]]: + key, value, op_type = f.key, f.value, f.type + ek_col = f"{self.alias}.entity_key" if self.alias else "entity_key" + + if op_type in _SQLITE_COMPARISON_OPS: + col, db_value = _sqlite_filter_col_and_val(value) + clause = ( + f"{ek_col} IN (SELECT entity_key FROM {_quote_id(self.table_name)} " + f"WHERE feature_name = ? AND {col} {_SQLITE_COMPARISON_OPS[op_type]} ?)" + ) + return clause, [key, db_value] + + if op_type == "in": + if not isinstance(value, list): + raise ValueError( + f"'in' filter requires a list value, got {type(value)}" + ) + col, _ = ( + _sqlite_filter_col_and_val(value[0]) if value else ("value_text", None) + ) + db_values = [_sqlite_filter_col_and_val(v)[1] for v in value] + placeholders = ", ".join(["?"] * len(value)) + clause = ( + f"{ek_col} IN (SELECT entity_key FROM {_quote_id(self.table_name)} " + f"WHERE feature_name = ? AND {col} IN ({placeholders}))" + ) + return clause, [key] + db_values + + if op_type == "nin": + if not isinstance(value, list): + raise ValueError( + f"'nin' filter requires a list value, got {type(value)}" + ) + col, _ = ( + _sqlite_filter_col_and_val(value[0]) if value else ("value_text", None) + ) + db_values = [_sqlite_filter_col_and_val(v)[1] for v in value] + placeholders = ", ".join(["?"] * len(value)) + clause = ( + f"{ek_col} IN (SELECT entity_key FROM {_quote_id(self.table_name)} " + f"WHERE feature_name = ? AND {col} NOT IN ({placeholders}))" + ) + return clause, [key] + db_values + + raise ValueError(f"Unknown comparison operator: {op_type}") + + def translate_compound(self, f: CompoundFilter) -> Tuple[str, List[Any]]: + if not f.filters: + return "", [] + parts: List[str] = [] + all_params: List[Any] = [] + for sub in f.filters: + sub_clause, sub_params = self._dispatch(sub) + parts.append(sub_clause) + all_params.extend(sub_params) + joiner = " AND " if f.type == "and" else " OR " + combined = "(" + joiner.join(parts) + ")" + return combined, all_params + + +def _sqlite_filter_col_and_val(value: Any) -> Tuple[str, Any]: + """Return the appropriate column name and DB-ready value for a filter value.""" + if isinstance(value, bool): + return "value_num", 1.0 if value else 0.0 + if isinstance(value, (int, float)): + return "value_num", float(value) + return "value_text", str(value) + + class SqliteOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): """Online store config for local (SQLite-based) store""" @@ -116,6 +214,8 @@ class SqliteOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): text_search_enabled: bool = False + enable_openai_compatible_store: bool = False + class SqliteOnlineStore(OnlineStore): """ @@ -126,6 +226,7 @@ class SqliteOnlineStore(OnlineStore): """ _conn: Optional[sqlite3.Connection] = None + _table_has_value_num: Optional[Dict[str, bool]] = None @staticmethod def _get_db_path(config: RepoConfig) -> str: @@ -150,6 +251,27 @@ def _get_conn(self, config: RepoConfig): return self._conn + def _check_table_has_value_num( + self, conn: sqlite3.Connection, table_name: str + ) -> bool: + """Check if the value_num column exists in the given table, with caching.""" + if self._table_has_value_num is None: + self._table_has_value_num = {} + if table_name in self._table_has_value_num: + return self._table_has_value_num[table_name] + + cur = conn.execute(f"PRAGMA table_info({_quote_id(table_name)})") + columns = {row[1] for row in cur.fetchall()} + exists = "value_num" in columns + self._table_has_value_num[table_name] = exists + return exists + + @staticmethod + def _filters_need_value_num( + filters: Union[ComparisonFilter, CompoundFilter], + ) -> bool: + return filters_contain_numeric_comparison(filters) + def online_write_batch( self, config: RepoConfig, @@ -167,6 +289,43 @@ def online_write_batch( conn = self._get_conn(config) project = config.project feature_type_dict = {f.name: f.dtype for f in table.features} + table_name = _table_id( + project, table, config.registry.enable_online_feature_view_versioning + ) + + enable_value_num = getattr( + config.online_store, "enable_openai_compatible_store", False + ) + has_value_num_col = False + if enable_value_num: + has_value_num_col = self._check_table_has_value_num(conn, table_name) + if not has_value_num_col: + logging.warning( + "enable_openai_compatible_store is True but value_num column " + "not found in table '%s'. Run `feast apply` to add it. " + "Writing without value_num.", + table_name, + ) + compute_value_num = has_value_num_col + columns = ["entity_key", "feature_name", "value", "value_text"] + if has_value_num_col: + columns.append("value_num") + if config.online_store.vector_enabled: + columns.append("vector_value") + columns.extend(["event_ts", "created_ts"]) + + pk_cols = {"entity_key", "feature_name"} + col_csv = ", ".join(columns) + placeholders = ", ".join(["?"] * len(columns)) + update_set = ", ".join( + f"{c} = excluded.{c}" for c in columns if c not in pk_cols + ) + upsert_sql = ( + f"INSERT INTO {_quote_id(table_name)} ({col_csv}) " + f"VALUES ({placeholders}) " + f"ON CONFLICT(entity_key, feature_name) DO UPDATE SET {update_set};" + ) + with conn: for entity_key, values, timestamp, created_ts in data: entity_key_bin = serialize_entity_key( @@ -177,12 +336,16 @@ def online_write_batch( if created_ts is not None: created_ts = to_naive_utc(created_ts) - table_name = _table_id( - project, - table, - config.registry.enable_online_feature_view_versioning, - ) for feature_name, val in values.items(): + value_text, value_num = extract_text_and_num(val, compute_value_num) + row: List[Any] = [ + entity_key_bin, + feature_name, + val.SerializeToString(), + value_text, + ] + if has_value_num_col: + row.append(value_num) if config.online_store.vector_enabled: if ( feature_type_dict.get(feature_name, None) @@ -198,43 +361,10 @@ def online_write_batch( ) # type: ignore else: val_bin = feast_value_type_to_python_type(val) - conn.execute( - f""" - INSERT INTO {table_name} (entity_key, feature_name, value, vector_value, event_ts, created_ts) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(entity_key, feature_name) DO UPDATE SET - value = excluded.value, - vector_value = excluded.vector_value, - event_ts = excluded.event_ts, - created_ts = excluded.created_ts; - """, - ( - entity_key_bin, # entity_key - feature_name, # feature_name - val.SerializeToString(), # value - val_bin, # vector_value - timestamp, # event_ts - created_ts, # created_ts - ), - ) - else: - conn.execute( - f""" - INSERT INTO {table_name} (entity_key, feature_name, value, event_ts, created_ts) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(entity_key, feature_name) DO UPDATE SET - value = excluded.value, - event_ts = excluded.event_ts, - created_ts = excluded.created_ts; - """, - ( - entity_key_bin, # entity_key - feature_name, # feature_name - val.SerializeToString(), # value - timestamp, # event_ts - created_ts, # created_ts - ), - ) + row.append(val_bin) + row.extend([timestamp, created_ts]) + + conn.execute(upsert_sql, tuple(row)) if progress: progress(1) @@ -261,7 +391,7 @@ def online_read( # Fetch all entities in one go cur.execute( f"SELECT entity_key, feature_name, value, event_ts " - f"FROM {_table_id(config.project, table, config.registry.enable_online_feature_view_versioning)} " + f"FROM {_quote_id(_table_id(config.project, table, config.registry.enable_online_feature_view_versioning))} " f"WHERE entity_key IN ({','.join('?' * len(entity_keys))}) " f"ORDER BY entity_key", serialized_entity_keys, @@ -300,19 +430,30 @@ def update( ): conn = self._get_conn(config) project = config.project + include_value_num = getattr( + config.online_store, "enable_openai_compatible_store", False + ) versioning = config.registry.enable_online_feature_view_versioning for table in tables_to_keep: + tbl = _table_id(project, table, versioning) + value_num_col = "value_num REAL," if include_value_num else "" conn.execute( - f"CREATE TABLE IF NOT EXISTS {_table_id(project, table, versioning)} (entity_key BLOB, feature_name TEXT, value BLOB, vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" + f"CREATE TABLE IF NOT EXISTS {_quote_id(tbl)} (entity_key BLOB, feature_name TEXT, value BLOB, value_text TEXT, {value_num_col} vector_value BLOB, event_ts timestamp, created_ts timestamp, PRIMARY KEY(entity_key, feature_name))" ) conn.execute( - f"CREATE INDEX IF NOT EXISTS {_table_id(project, table, versioning)}_ek ON {_table_id(project, table, versioning)} (entity_key);" + f"CREATE INDEX IF NOT EXISTS {_quote_id(tbl + '_ek')} ON {_quote_id(tbl)} (entity_key);" ) + _alter_table_add_column_if_missing(conn, tbl, "value_text", "TEXT") + if include_value_num: + _alter_table_add_column_if_missing(conn, tbl, "value_num", "REAL") + if self._table_has_value_num is None: + self._table_has_value_num = {} + self._table_has_value_num[tbl] = True for table in tables_to_delete: conn.execute( - f"DROP TABLE IF EXISTS {_table_id(project, table, versioning)}" + f"DROP TABLE IF EXISTS {_quote_id(_table_id(project, table, versioning))}" ) def plan( @@ -320,6 +461,9 @@ def plan( ) -> List[InfraObject]: project = config.project versioning = config.registry.enable_online_feature_view_versioning + include_value_num = getattr( + config.online_store, "enable_openai_compatible_store", False + ) infra_objects: List[InfraObject] = [ SqliteTable( @@ -329,6 +473,7 @@ def plan( FeatureView.from_proto(view), versioning, ), + include_value_num=include_value_num, ) for view in [ *desired_registry_proto.feature_views, @@ -431,9 +576,10 @@ def retrieve_online_documents( cur.execute( f""" INSERT INTO vec_table(rowid, vector_value) - select rowid, vector_value from {table_name} - where feature_name = "{vector_field}" - """ + select rowid, vector_value from {_quote_id(table_name)} + where feature_name = ? + """, + (vector_field,), ) cur.execute( f""" @@ -443,7 +589,7 @@ def retrieve_online_documents( """ ) - # Have to join this with the {table_name} to get the feature name and entity_key + # Have to join this with the main table to get the feature name and entity_key # Also the `top_k` doesn't appear to be working for some reason cur.execute( f""" @@ -463,7 +609,7 @@ def retrieve_online_documents( order by distance limit ? ) f - left join {table_name} fv + left join {_quote_id(table_name)} fv on f.rowid = fv.rowid """, (query_embedding_bin, top_k), @@ -501,10 +647,11 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - query: Optional[List[float]], + embedding: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -519,7 +666,7 @@ def retrieve_online_documents_v2( config: Feast configuration object table: FeatureView object as the table to search requested_features: List of requested features to retrieve - query: Query embedding to search for (optional) + embedding: Query embedding to search for (optional) top_k: Number of items to return distance_metric: Distance metric to use (optional) query_string: The query string to search for using keyword search (bm25) (optional) @@ -534,6 +681,16 @@ def retrieve_online_documents_v2( "You must enable either vector search or text search in the online store config" ) + if filters is not None: + if not getattr( + config.online_store, "enable_openai_compatible_store", False + ): + raise ValueError( + "Metadata filtering requires `enable_openai_compatible_store: true` " + "in your online store config. After setting it, run `feast apply` " + "to update the database schema." + ) + conn = self._get_conn(config) cur = conn.cursor() @@ -546,8 +703,19 @@ def retrieve_online_documents_v2( ) vector_field = _get_vector_field(table) + if filters is not None and self._filters_need_value_num(filters): + if not self._check_table_has_value_num(conn, table_name): + raise ValueError( + "Numerical filtering requires the `value_num` column. " + "Run `feast apply` to add it." + ) + + filter_clause, filter_params = SqliteFilterTranslator( + table_name, alias="fv2" + ).translate(filters) + if online_store.vector_enabled: - query_embedding_bin = serialize_f32(query, vector_field_length) # type: ignore + query_embedding_bin = serialize_f32(embedding, vector_field_length) # type: ignore cur.execute( f""" CREATE VIRTUAL TABLE IF NOT EXISTS vec_table using vec0( @@ -558,16 +726,16 @@ def retrieve_online_documents_v2( cur.execute( f""" INSERT INTO vec_table (rowid, vector_value) - select rowid, vector_value from {table_name} - where feature_name = "{vector_field}" - """ + select rowid, vector_value from {_quote_id(table_name)} + where feature_name = ? + """, + (vector_field,), ) elif online_store.text_search_enabled: string_field_list = [ f.name for f in table.features if f.dtype == PrimitiveFeastType.STRING ] string_fields = ", ".join(string_field_list) - # TODO: swap this for a value configurable in each Field() BM25_DEFAULT_WEIGHTS = ", ".join( [ str(1.0) @@ -586,6 +754,9 @@ def retrieve_online_documents_v2( table_name, string_field_list ) cur.execute(insert_query) + filter_clause, filter_params = SqliteFilterTranslator( + table_name, alias="fv" + ).translate(filters) else: raise ValueError( @@ -593,6 +764,12 @@ def retrieve_online_documents_v2( ) if online_store.vector_enabled: + where_parts = ["fv2.feature_name != ?"] + vector_params = [vector_field] + if filter_clause: + where_parts.append(filter_clause) + where_sql = " AND ".join(where_parts) + cur.execute( f""" select @@ -613,29 +790,33 @@ def retrieve_online_documents_v2( order by distance limit ? ) f - left join {table_name} fv + left join {_quote_id(table_name)} fv on f.rowid = fv.rowid - left join {table_name} fv2 + left join {_quote_id(table_name)} fv2 on fv.entity_key = fv2.entity_key - where fv2.feature_name != "{vector_field}" + where {where_sql} """, - ( - query_embedding_bin, - top_k, - ), + [query_embedding_bin, top_k] + vector_params + filter_params, ) elif online_store.text_search_enabled: + where_parts_text = [] + if filter_clause: + where_parts_text.append(filter_clause) + where_sql_text = ( + "where " + " AND ".join(where_parts_text) if where_parts_text else "" + ) + cur.execute( f""" - select - fv.entity_key, - fv.feature_name, - fv.value, - fv.vector_value, - f.distance, - fv.event_ts, - fv.created_ts - from {table_name} fv + select + fv.entity_key, + fv.feature_name, + fv.value, + fv.vector_value, + f.distance, + fv.event_ts, + fv.created_ts + from {_quote_id(table_name)} fv inner join ( select fv_rowid, @@ -646,8 +827,9 @@ def retrieve_online_documents_v2( where search_table match ? order by distance limit ? ) f on f.entity_key = fv.entity_key + {where_sql_text} """, - (query_string, top_k), + [query_string, top_k] + filter_params, ) else: @@ -743,6 +925,36 @@ def _initialize_conn( return db +def _alter_table_add_column_if_missing( + conn: sqlite3.Connection, + table_name: str, + column_name: str, + column_type: str, +) -> None: + """Add a column to an existing SQLite table, ignoring if it already exists. + + SQLite's ALTER TABLE ADD COLUMN doesn't support IF NOT EXISTS, so we + catch the specific OperationalError for duplicate columns and re-raise + anything else (connection failures, disk errors, etc.). + """ + try: + conn.execute( + f"ALTER TABLE {_quote_id(table_name)} ADD COLUMN {_quote_id(column_name)} {column_type}" + ) + except sqlite3.OperationalError as e: + if "duplicate column name" not in str(e).lower(): + raise + + +def _quote_id(identifier: str) -> str: + """Quote a SQLite identifier to prevent SQL injection. + + Uses the standard SQL double-quote mechanism: any embedded + double-quote characters are escaped by doubling them. + """ + return '"' + identifier.replace('"', '""') + '"' + + def _table_id(project: str, table: Any, enable_versioning: bool = False) -> str: return compute_table_id(project, table, enable_versioning) @@ -759,11 +971,13 @@ class SqliteTable(InfraObject): path: str conn: sqlite3.Connection + _include_value_num: bool - def __init__(self, path: str, name: str): + def __init__(self, path: str, name: str, include_value_num: bool = False): super().__init__(name) self.path = path self.conn = _initialize_conn(path) + self._include_value_num = include_value_num def to_infra_object_proto(self) -> InfraObjectProto: sqlite_table_proto = self.to_proto() @@ -801,12 +1015,15 @@ def update(self): sqlite_vec.load(self.conn) except ModuleNotFoundError: logging.warning("Cannot use sqlite_vec for vector search") + value_num_col = "value_num REAL," if self._include_value_num else "" self.conn.execute( f""" CREATE TABLE IF NOT EXISTS {self.name} ( entity_key BLOB, feature_name TEXT, value BLOB, + value_text TEXT, + {value_num_col} vector_value BLOB, event_ts timestamp, created_ts timestamp, @@ -817,6 +1034,11 @@ def update(self): self.conn.execute( f"CREATE INDEX IF NOT EXISTS {self.name}_ek ON {self.name} (entity_key);" ) + _alter_table_add_column_if_missing(self.conn, self.name, "value_text", "TEXT") + if self._include_value_num: + _alter_table_add_column_if_missing( + self.conn, self.name, "value_num", "REAL" + ) def teardown(self): self.conn.execute(f"DROP TABLE IF EXISTS {self.name}") @@ -854,13 +1076,14 @@ def _generate_bm25_search_insert_query( """ _string_fields = ", ".join(string_field_list) query = f"INSERT INTO search_table (entity_key, fv_rowid, {_string_fields})\nSELECT\n\tDISTINCT fv0.entity_key,\n\tfv0.rowid as fv_rowid" - from_query = f"\nFROM (select rowid, * from {table_name} where feature_name = '{string_field_list[0]}') fv0" + quoted_table = _quote_id(table_name) + from_query = f"\nFROM (select rowid, * from {quoted_table} where feature_name = '{string_field_list[0]}') fv0" for i, string_field in enumerate(string_field_list): query += f"\n\t,fv{i}.value as {string_field}" if i > 0: from_query += ( - f"\nLEFT JOIN (select rowid, * from {table_name} where feature_name = '{string_field}') fv{i}" + f"\nLEFT JOIN (select rowid, * from {quoted_table} where feature_name = '{string_field}') fv{i}" + f"\n\tON fv0.entity_key = fv{i}.entity_key" ) diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 93f38567376..85c706b2307 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -24,6 +24,7 @@ from feast.feature_logging import FeatureServiceLoggingSource from feast.feature_service import FeatureService from feast.feature_view import FeatureView +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.infra.common.materialization_job import ( MaterializationJobStatus, MaterializationTask, @@ -322,23 +323,25 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: Optional[List[str]], - query: Optional[List[float]], + embedding: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List: result = [] if self.online_store: result = self.online_store.retrieve_online_documents_v2( - config, - table, - requested_features, - query, - top_k, - distance_metric, - query_string, - include_feature_view_version_metadata, + config=config, + table=table, + requested_features=requested_features, + embedding=embedding, + top_k=top_k, + distance_metric=distance_metric, + query_string=query_string, + filters=filters, + include_feature_view_version_metadata=include_feature_view_version_metadata, ) return result diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 705556623df..c3912cd9f55 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -23,6 +23,7 @@ from feast.data_source import DataSource from feast.entity import Entity from feast.feature_view import FeatureView +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.importer import import_class from feast.infra.infra_object import Infra from feast.infra.offline_stores.offline_store import OfflineStore, RetrievalJob @@ -467,10 +468,11 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - query: Optional[List[float]], + embedding: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ @@ -490,6 +492,8 @@ def retrieve_online_documents_v2( query: The query embedding to search for (optional). top_k: The number of documents to return. query_string: The query string to search for using keyword search (bm25) (optional) + filters: Optional metadata filters (ComparisonFilter or CompoundFilter) + to narrow results before ranking. Returns: A list of dictionaries, where each dictionary contains the datetime, entitykey, and a dictionary diff --git a/sdk/python/feast/infra/registry/base_registry.py b/sdk/python/feast/infra/registry/base_registry.py index a0d98d5d2c2..4256c82e50d 100644 --- a/sdk/python/feast/infra/registry/base_registry.py +++ b/sdk/python/feast/infra/registry/base_registry.py @@ -698,6 +698,8 @@ def list_saved_datasets( project: str, allow_cache: bool = False, tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: """ Retrieves a list of all saved datasets in specified project @@ -706,6 +708,8 @@ def list_saved_datasets( project: Feast project allow_cache: Whether to allow returning this dataset from a cached registry tags: Filter by tags + namespace: Filter by logical namespace grouping + collection: Filter by collection sub-grouping within namespace Returns: Returns the list of SavedDatasets diff --git a/sdk/python/feast/infra/registry/caching_registry.py b/sdk/python/feast/infra/registry/caching_registry.py index 6780a2ece69..81fc11ed025 100644 --- a/sdk/python/feast/infra/registry/caching_registry.py +++ b/sdk/python/feast/infra/registry/caching_registry.py @@ -344,7 +344,11 @@ def get_saved_dataset( @abstractmethod def _list_saved_datasets( - self, project: str, tags: Optional[dict[str, str]] = None + self, + project: str, + tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: pass @@ -353,13 +357,21 @@ def list_saved_datasets( project: str, allow_cache: bool = False, tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: if allow_cache: self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_saved_datasets( - self.cached_registry_proto, project, tags + self.cached_registry_proto, + project, + tags, + namespace=namespace, + collection=collection, ) - return self._list_saved_datasets(project, tags) + return self._list_saved_datasets( + project, tags, namespace=namespace, collection=collection + ) @abstractmethod def _get_validation_reference(self, name: str, project: str) -> ValidationReference: diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py index de5d199555b..2be5e7f2cb6 100644 --- a/sdk/python/feast/infra/registry/proto_registry_utils.py +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -418,14 +418,23 @@ def list_data_sources( @registry_proto_cache_with_tags def list_saved_datasets( - registry_proto: RegistryProto, project: str, tags: Optional[dict[str, str]] + registry_proto: RegistryProto, + project: str, + tags: Optional[dict[str, str]], + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: saved_datasets = [] for saved_dataset in registry_proto.saved_datasets: - if saved_dataset.spec.project == project and utils.has_all_tags( - saved_dataset.spec.tags, tags - ): - saved_datasets.append(SavedDataset.from_proto(saved_dataset)) + if saved_dataset.spec.project != project: + continue + if not utils.has_all_tags(saved_dataset.spec.tags, tags): + continue + if namespace is not None and saved_dataset.spec.namespace != namespace: + continue + if collection is not None and saved_dataset.spec.collection != collection: + continue + saved_datasets.append(SavedDataset.from_proto(saved_dataset)) return saved_datasets diff --git a/sdk/python/feast/infra/registry/registry.py b/sdk/python/feast/infra/registry/registry.py index f09f05e971f..78994f5b8c3 100644 --- a/sdk/python/feast/infra/registry/registry.py +++ b/sdk/python/feast/infra/registry/registry.py @@ -1305,11 +1305,19 @@ def list_saved_datasets( project: str, allow_cache: bool = False, tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: registry_proto = self._get_registry_proto( project=project, allow_cache=allow_cache ) - return proto_registry_utils.list_saved_datasets(registry_proto, project, tags) + return proto_registry_utils.list_saved_datasets( + registry_proto, + project, + tags, + namespace=namespace, + collection=collection, + ) def delete_saved_dataset(self, name: str, project: str, commit: bool = True): self._prepare_registry_for_changes(project) diff --git a/sdk/python/feast/infra/registry/remote.py b/sdk/python/feast/infra/registry/remote.py index 287eb3fda17..43550a49882 100644 --- a/sdk/python/feast/infra/registry/remote.py +++ b/sdk/python/feast/infra/registry/remote.py @@ -511,9 +511,15 @@ def list_saved_datasets( project: str, allow_cache: bool = False, tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: request = RegistryServer_pb2.ListSavedDatasetsRequest( - project=project, allow_cache=allow_cache, tags=tags + project=project, + allow_cache=allow_cache, + tags=tags, + namespace=namespace or "", + collection=collection or "", ) response = self.stub.ListSavedDatasets(request) return [ diff --git a/sdk/python/feast/infra/registry/snowflake.py b/sdk/python/feast/infra/registry/snowflake.py index 5590e1b7574..c04ea65b7df 100644 --- a/sdk/python/feast/infra/registry/snowflake.py +++ b/sdk/python/feast/infra/registry/snowflake.py @@ -945,13 +945,19 @@ def list_saved_datasets( project: str, allow_cache: bool = False, tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, ) -> List[SavedDataset]: if allow_cache: registry_proto = self._refresh_cached_registry_if_necessary() return proto_registry_utils.list_saved_datasets( - registry_proto, project, tags + registry_proto, + project, + tags, + namespace=namespace, + collection=collection, ) - return self._list_objects( + results = self._list_objects( "SAVED_DATASETS", project, SavedDatasetProto, @@ -959,6 +965,11 @@ def list_saved_datasets( "SAVED_DATASET_PROTO", tags=tags, ) + if namespace is not None: + results = [sd for sd in results if sd.namespace == namespace] + if collection is not None: + results = [sd for sd in results if sd.collection == collection] + return results def list_stream_feature_views( self, diff --git a/sdk/python/feast/infra/registry/sql.py b/sdk/python/feast/infra/registry/sql.py index edd89347be2..32fc7dd07f0 100644 --- a/sdk/python/feast/infra/registry/sql.py +++ b/sdk/python/feast/infra/registry/sql.py @@ -1076,9 +1076,14 @@ def _list_feature_views( ) def _list_saved_datasets( - self, project: str, tags: Optional[dict[str, str]] = None, **kwargs + self, + project: str, + tags: Optional[dict[str, str]] = None, + namespace: Optional[str] = None, + collection: Optional[str] = None, + **kwargs, ) -> List[SavedDataset]: - return self._list_objects( + results = self._list_objects( saved_datasets, project, SavedDatasetProto, @@ -1087,6 +1092,11 @@ def _list_saved_datasets( tags=tags, **kwargs, ) + if namespace is not None: + results = [sd for sd in results if sd.namespace == namespace] + if collection is not None: + results = [sd for sd in results if sd.collection == collection] + return results def _list_on_demand_feature_views( self, project: str, tags: Optional[dict[str, str]], **kwargs diff --git a/sdk/python/feast/protos/feast/core/SavedDataset_pb2.py b/sdk/python/feast/protos/feast/core/SavedDataset_pb2.py index fe1e2d49eac..8e394dec7ec 100644 --- a/sdk/python/feast/protos/feast/core/SavedDataset_pb2.py +++ b/sdk/python/feast/protos/feast/core/SavedDataset_pb2.py @@ -16,7 +16,7 @@ from feast.protos.feast.core import DataSource_pb2 as feast_dot_core_dot_DataSource__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x66\x65\x61st/core/SavedDataset.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\"\xa5\x02\n\x10SavedDatasetSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x66\x65\x61tures\x18\x03 \x03(\t\x12\x11\n\tjoin_keys\x18\x04 \x03(\t\x12\x1a\n\x12\x66ull_feature_names\x18\x05 \x01(\x08\x12\x30\n\x07storage\x18\x06 \x01(\x0b\x32\x1f.feast.core.SavedDatasetStorage\x12\x1c\n\x14\x66\x65\x61ture_service_name\x18\x08 \x01(\t\x12\x34\n\x04tags\x18\x07 \x03(\x0b\x32&.feast.core.SavedDatasetSpec.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa9\x04\n\x13SavedDatasetStorage\x12:\n\x0c\x66ile_storage\x18\x04 \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_storage\x18\x05 \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12\x42\n\x10redshift_storage\x18\x06 \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12\x44\n\x11snowflake_storage\x18\x07 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12<\n\rtrino_storage\x18\x08 \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12<\n\rspark_storage\x18\t \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12\x44\n\x0e\x63ustom_storage\x18\n \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12>\n\x0e\x61thena_storage\x18\x0b \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x42\x06\n\x04kind\"\xf7\x01\n\x10SavedDatasetMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13min_event_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13max_event_timestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"f\n\x0cSavedDataset\x12*\n\x04spec\x18\x01 \x01(\x0b\x32\x1c.feast.core.SavedDatasetSpec\x12*\n\x04meta\x18\x02 \x01(\x0b\x32\x1c.feast.core.SavedDatasetMetaBV\n\x10\x66\x65\x61st.proto.coreB\x11SavedDatasetProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x66\x65\x61st/core/SavedDataset.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\"\xe1\x02\n\x10SavedDatasetSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x66\x65\x61tures\x18\x03 \x03(\t\x12\x11\n\tjoin_keys\x18\x04 \x03(\t\x12\x1a\n\x12\x66ull_feature_names\x18\x05 \x01(\x08\x12\x30\n\x07storage\x18\x06 \x01(\x0b\x32\x1f.feast.core.SavedDatasetStorage\x12\x1c\n\x14\x66\x65\x61ture_service_name\x18\x08 \x01(\t\x12\x34\n\x04tags\x18\x07 \x03(\x0b\x32&.feast.core.SavedDatasetSpec.TagsEntry\x12\x11\n\tnamespace\x18\t \x01(\t\x12\x12\n\ncollection\x18\n \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x0b \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa9\x04\n\x13SavedDatasetStorage\x12:\n\x0c\x66ile_storage\x18\x04 \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_storage\x18\x05 \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12\x42\n\x10redshift_storage\x18\x06 \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12\x44\n\x11snowflake_storage\x18\x07 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12<\n\rtrino_storage\x18\x08 \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12<\n\rspark_storage\x18\t \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12\x44\n\x0e\x63ustom_storage\x18\n \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12>\n\x0e\x61thena_storage\x18\x0b \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x42\x06\n\x04kind\"\xf7\x01\n\x10SavedDatasetMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13min_event_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x37\n\x13max_event_timestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"f\n\x0cSavedDataset\x12*\n\x04spec\x18\x01 \x01(\x0b\x32\x1c.feast.core.SavedDatasetSpec\x12*\n\x04meta\x18\x02 \x01(\x0b\x32\x1c.feast.core.SavedDatasetMetaBV\n\x10\x66\x65\x61st.proto.coreB\x11SavedDatasetProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -27,13 +27,13 @@ _globals['_SAVEDDATASETSPEC_TAGSENTRY']._options = None _globals['_SAVEDDATASETSPEC_TAGSENTRY']._serialized_options = b'8\001' _globals['_SAVEDDATASETSPEC']._serialized_start=108 - _globals['_SAVEDDATASETSPEC']._serialized_end=401 - _globals['_SAVEDDATASETSPEC_TAGSENTRY']._serialized_start=358 - _globals['_SAVEDDATASETSPEC_TAGSENTRY']._serialized_end=401 - _globals['_SAVEDDATASETSTORAGE']._serialized_start=404 - _globals['_SAVEDDATASETSTORAGE']._serialized_end=957 - _globals['_SAVEDDATASETMETA']._serialized_start=960 - _globals['_SAVEDDATASETMETA']._serialized_end=1207 - _globals['_SAVEDDATASET']._serialized_start=1209 - _globals['_SAVEDDATASET']._serialized_end=1311 + _globals['_SAVEDDATASETSPEC']._serialized_end=461 + _globals['_SAVEDDATASETSPEC_TAGSENTRY']._serialized_start=418 + _globals['_SAVEDDATASETSPEC_TAGSENTRY']._serialized_end=461 + _globals['_SAVEDDATASETSTORAGE']._serialized_start=464 + _globals['_SAVEDDATASETSTORAGE']._serialized_end=1017 + _globals['_SAVEDDATASETMETA']._serialized_start=1020 + _globals['_SAVEDDATASETMETA']._serialized_end=1267 + _globals['_SAVEDDATASET']._serialized_start=1269 + _globals['_SAVEDDATASET']._serialized_end=1371 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/SavedDataset_pb2.pyi b/sdk/python/feast/protos/feast/core/SavedDataset_pb2.pyi index 47525b64ede..2fbf5e05271 100644 --- a/sdk/python/feast/protos/feast/core/SavedDataset_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/SavedDataset_pb2.pyi @@ -58,6 +58,9 @@ class SavedDatasetSpec(google.protobuf.message.Message): STORAGE_FIELD_NUMBER: builtins.int FEATURE_SERVICE_NAME_FIELD_NUMBER: builtins.int TAGS_FIELD_NUMBER: builtins.int + NAMESPACE_FIELD_NUMBER: builtins.int + COLLECTION_FIELD_NUMBER: builtins.int + DESCRIPTION_FIELD_NUMBER: builtins.int name: builtins.str """Name of the dataset. Must be unique since it's possible to overwrite dataset by name""" project: builtins.str @@ -77,6 +80,18 @@ class SavedDatasetSpec(google.protobuf.message.Message): @property def tags(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: """User defined metadata""" + namespace: builtins.str + """Optional logical namespace for hierarchical grouping. + Maps to the top-level prefix in Iceberg REST Catalog API. + Empty string means not set (no namespace scoping). + """ + collection: builtins.str + """Optional sub-grouping within a namespace. + Maps to the namespace level in Iceberg REST Catalog API. + Empty string means not set (dataset sits directly under namespace). + """ + description: builtins.str + """Description of the saved dataset.""" def __init__( self, *, @@ -88,9 +103,12 @@ class SavedDatasetSpec(google.protobuf.message.Message): storage: global___SavedDatasetStorage | None = ..., feature_service_name: builtins.str = ..., tags: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + namespace: builtins.str = ..., + collection: builtins.str = ..., + description: builtins.str = ..., ) -> None: ... def HasField(self, field_name: typing_extensions.Literal["storage", b"storage"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["feature_service_name", b"feature_service_name", "features", b"features", "full_feature_names", b"full_feature_names", "join_keys", b"join_keys", "name", b"name", "project", b"project", "storage", b"storage", "tags", b"tags"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["collection", b"collection", "description", b"description", "feature_service_name", b"feature_service_name", "features", b"features", "full_feature_names", b"full_feature_names", "join_keys", b"join_keys", "name", b"name", "namespace", b"namespace", "project", b"project", "storage", b"storage", "tags", b"tags"]) -> None: ... global___SavedDatasetSpec = SavedDatasetSpec diff --git a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py index 25766701899..95e5c449912 100644 --- a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py +++ b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py @@ -29,7 +29,7 @@ from feast.protos.feast.core import Project_pb2 as feast_dot_core_dot_Project__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1a\x66\x65\x61st/core/LabelView.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"/\n\x10PaginationParams\x12\x0c\n\x04page\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\"4\n\rSortingParams\x12\x0f\n\x07sort_by\x18\x01 \x01(\t\x12\x12\n\nsort_order\x18\x02 \x01(\t\"\x83\x01\n\x12PaginationMetadata\x12\x0c\n\x04page\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\x12\x13\n\x0btotal_count\x18\x03 \x01(\x05\x12\x13\n\x0btotal_pages\x18\x04 \x01(\x05\x12\x10\n\x08has_next\x18\x05 \x01(\x08\x12\x14\n\x0chas_previous\x18\x06 \x01(\x08\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x8b\x02\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"t\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x91\x02\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xae\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12+\n\nlabel_view\x18\x06 \x01(\x0b\x32\x15.feast.core.LabelViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x93\x02\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x82\x01\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x83\x02\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12+\n\nlabel_view\x18\x04 \x01(\x0b\x32\x15.feast.core.LabelViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\x9b\x03\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x12\x0e\n\x06\x65ntity\x18\x04 \x01(\t\x12\x0f\n\x07\x66\x65\x61ture\x18\x05 \x01(\t\x12\x17\n\x0f\x66\x65\x61ture_service\x18\x06 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x07 \x01(\t\x12\x34\n\npagination\x18\x08 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\t \x01(\x0b\x32\x1d.feast.registry.SortingParams\x12\x31\n\rupdated_since\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x9f\x02\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x95\x01\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa3\x02\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9c\x01\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"I\n\x13GetLabelViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x8f\x02\n\x15ListLabelViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12=\n\x04tags\x18\x03 \x03(\x0b\x32/.feast.registry.ListLabelViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"|\n\x16ListLabelViewsResponse\x12*\n\x0blabel_views\x18\x01 \x03(\x0b\x32\x15.feast.core.LabelView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x02\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x04 \x01(\t\x12\x34\n\npagination\x18\x05 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x06 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8b\x01\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x95\x02\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x85\x01\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd0\x03\n!CreateDatasetFromRetrievalRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x1c\n\x14\x66\x65\x61ture_service_name\x18\x03 \x01(\t\x12\x10\n\x08\x66\x65\x61tures\x18\x04 \x03(\t\x12\x1a\n\x12\x65ntity_source_type\x18\x05 \x01(\t\x12\x1a\n\x12\x65ntity_source_path\x18\x06 \x01(\t\x12\x13\n\x0b\x65ntity_keys\x18\x08 \x03(\t\x12\x15\n\rentity_values\x18\t \x01(\t\x12\x12\n\nstart_date\x18\n \x01(\t\x12\x10\n\x08\x65nd_date\x18\x0b \x01(\t\x12\x15\n\rextra_columns\x18\x0c \x01(\t\x12\x14\n\x0cstorage_type\x18\r \x01(\t\x12\x14\n\x0cstorage_path\x18\x0e \x01(\t\x12I\n\x04tags\x18\x0f \x03(\x0b\x32;.feast.registry.CreateDatasetFromRetrievalRequest.TagsEntry\x12\x17\n\x0f\x61llow_overwrite\x18\x10 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"D\n\"CreateDatasetFromRetrievalResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\t\"E\n\x15GetDatasetDataRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05\"\x1c\n\nTabularRow\x12\x0e\n\x06values\x18\x01 \x03(\t\"|\n\x16GetDatasetDataResponse\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\x12(\n\x04rows\x18\x02 \x03(\x0b\x32\x1a.feast.registry.TabularRow\x12\x12\n\ntotal_rows\x18\x03 \x01(\x05\x12\x13\n\x0bsample_size\x18\x04 \x01(\x05\",\n\x1aGetDatasetJobStatusRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9d\x01\n\x1bGetDatasetJobStatusResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x14\n\x0c\x64\x61taset_name\x18\x02 \x01(\t\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0e\n\x06status\x18\x04 \x01(\t\x12\x12\n\ncreated_at\x18\x05 \x01(\t\x12\x14\n\x0c\x63ompleted_at\x18\x06 \x01(\t\x12\r\n\x05\x65rror\x18\x07 \x01(\t\"@\n\x16ListDatasetJobsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x15\n\rstatus_filter\x18\x02 \x01(\t\"T\n\x17ListDatasetJobsResponse\x12\x39\n\x04jobs\x18\x01 \x03(\x0b\x32+.feast.registry.GetDatasetJobStatusResponse\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa3\x02\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x91\x02\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"~\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\xfa\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x12\x34\n\npagination\x18\x03 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x04 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"u\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"-\n\x0f\x45ntityReference\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"r\n\x0e\x45ntityRelation\x12/\n\x06source\x18\x01 \x01(\x0b\x32\x1f.feast.registry.EntityReference\x12/\n\x06target\x18\x02 \x01(\x0b\x32\x1f.feast.registry.EntityReference\"\xdf\x01\n\x19GetRegistryLineageRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x1a\n\x12\x66ilter_object_type\x18\x03 \x01(\t\x12\x1a\n\x12\x66ilter_object_name\x18\x04 \x01(\t\x12\x34\n\npagination\x18\x05 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x06 \x01(\x0b\x32\x1d.feast.registry.SortingParams\"\xa8\x02\n\x1aGetRegistryLineageResponse\x12\x35\n\rrelationships\x18\x01 \x03(\x0b\x32\x1e.feast.registry.EntityRelation\x12>\n\x16indirect_relationships\x18\x02 \x03(\x0b\x32\x1e.feast.registry.EntityRelation\x12\x44\n\x18relationships_pagination\x18\x03 \x01(\x0b\x32\".feast.registry.PaginationMetadata\x12M\n!indirect_relationships_pagination\x18\x04 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"\xef\x01\n\x1dGetObjectRelationshipsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0bobject_type\x18\x02 \x01(\t\x12\x13\n\x0bobject_name\x18\x03 \x01(\t\x12\x18\n\x10include_indirect\x18\x04 \x01(\x08\x12\x13\n\x0b\x61llow_cache\x18\x05 \x01(\x08\x12\x34\n\npagination\x18\x06 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x07 \x01(\x0b\x32\x1d.feast.registry.SortingParams\"\x8f\x01\n\x1eGetObjectRelationshipsResponse\x12\x35\n\rrelationships\x18\x01 \x03(\x0b\x32\x1e.feast.registry.EntityRelation\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"\xbe\x02\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\r\n\x05owner\x18\x05 \x01(\t\x12\x35\n\x11\x63reated_timestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x04tags\x18\x08 \x03(\x0b\x32!.feast.registry.Feature.TagsEntry\x12\x0c\n\x04kind\x18\t \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd3\x01\n\x13ListFeaturesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x06 \x01(\x08\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x12\x0c\n\x04kind\x18\x07 \x01(\t\"y\n\x14ListFeaturesResponse\x12)\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x17.feast.registry.Feature\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"]\n\x11GetFeatureRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x04 \x01(\x08\x32\xd2(\n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12L\n\x0cGetLabelView\x12#.feast.registry.GetLabelViewRequest\x1a\x15.feast.core.LabelView\"\x00\x12\x61\n\x0eListLabelViews\x12%.feast.registry.ListLabelViewsRequest\x1a&.feast.registry.ListLabelViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x85\x01\n\x1a\x43reateDatasetFromRetrieval\x12\x31.feast.registry.CreateDatasetFromRetrievalRequest\x1a\x32.feast.registry.CreateDatasetFromRetrievalResponse\"\x00\x12\x61\n\x0eGetDatasetData\x12%.feast.registry.GetDatasetDataRequest\x1a&.feast.registry.GetDatasetDataResponse\"\x00\x12p\n\x13GetDatasetJobStatus\x12*.feast.registry.GetDatasetJobStatusRequest\x1a+.feast.registry.GetDatasetJobStatusResponse\"\x00\x12\x64\n\x0fListDatasetJobs\x12&.feast.registry.ListDatasetJobsRequest\x1a\'.feast.registry.ListDatasetJobsResponse\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x12m\n\x12GetRegistryLineage\x12).feast.registry.GetRegistryLineageRequest\x1a*.feast.registry.GetRegistryLineageResponse\"\x00\x12y\n\x16GetObjectRelationships\x12-.feast.registry.GetObjectRelationshipsRequest\x1a..feast.registry.GetObjectRelationshipsResponse\"\x00\x12[\n\x0cListFeatures\x12#.feast.registry.ListFeaturesRequest\x1a$.feast.registry.ListFeaturesResponse\"\x00\x12J\n\nGetFeature\x12!.feast.registry.GetFeatureRequest\x1a\x17.feast.registry.Feature\"\x00\x42\x35Z3github.com/feast-dev/feast/go/protos/feast/registryb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1a\x66\x65\x61st/core/LabelView.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"/\n\x10PaginationParams\x12\x0c\n\x04page\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\"4\n\rSortingParams\x12\x0f\n\x07sort_by\x18\x01 \x01(\t\x12\x12\n\nsort_order\x18\x02 \x01(\t\"\x83\x01\n\x12PaginationMetadata\x12\x0c\n\x04page\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\x12\x13\n\x0btotal_count\x18\x03 \x01(\x05\x12\x13\n\x0btotal_pages\x18\x04 \x01(\x05\x12\x10\n\x08has_next\x18\x05 \x01(\x08\x12\x14\n\x0chas_previous\x18\x06 \x01(\x08\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x8b\x02\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"t\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x91\x02\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xae\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12+\n\nlabel_view\x18\x06 \x01(\x0b\x32\x15.feast.core.LabelViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x93\x02\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x82\x01\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x83\x02\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12+\n\nlabel_view\x18\x04 \x01(\x0b\x32\x15.feast.core.LabelViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\x9b\x03\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x12\x0e\n\x06\x65ntity\x18\x04 \x01(\t\x12\x0f\n\x07\x66\x65\x61ture\x18\x05 \x01(\t\x12\x17\n\x0f\x66\x65\x61ture_service\x18\x06 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x07 \x01(\t\x12\x34\n\npagination\x18\x08 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\t \x01(\x0b\x32\x1d.feast.registry.SortingParams\x12\x31\n\rupdated_since\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x9f\x02\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x95\x01\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa3\x02\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9c\x01\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"I\n\x13GetLabelViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x8f\x02\n\x15ListLabelViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12=\n\x04tags\x18\x03 \x03(\x0b\x32/.feast.registry.ListLabelViewsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"|\n\x16ListLabelViewsResponse\x12*\n\x0blabel_views\x18\x01 \x03(\x0b\x32\x15.feast.core.LabelView\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x02\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x04 \x01(\t\x12\x34\n\npagination\x18\x05 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x06 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8b\x01\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbc\x02\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x12\x11\n\tnamespace\x18\x06 \x01(\t\x12\x12\n\ncollection\x18\x07 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x85\x01\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd0\x03\n!CreateDatasetFromRetrievalRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x1c\n\x14\x66\x65\x61ture_service_name\x18\x03 \x01(\t\x12\x10\n\x08\x66\x65\x61tures\x18\x04 \x03(\t\x12\x1a\n\x12\x65ntity_source_type\x18\x05 \x01(\t\x12\x1a\n\x12\x65ntity_source_path\x18\x06 \x01(\t\x12\x13\n\x0b\x65ntity_keys\x18\x08 \x03(\t\x12\x15\n\rentity_values\x18\t \x01(\t\x12\x12\n\nstart_date\x18\n \x01(\t\x12\x10\n\x08\x65nd_date\x18\x0b \x01(\t\x12\x15\n\rextra_columns\x18\x0c \x01(\t\x12\x14\n\x0cstorage_type\x18\r \x01(\t\x12\x14\n\x0cstorage_path\x18\x0e \x01(\t\x12I\n\x04tags\x18\x0f \x03(\x0b\x32;.feast.registry.CreateDatasetFromRetrievalRequest.TagsEntry\x12\x17\n\x0f\x61llow_overwrite\x18\x10 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"D\n\"CreateDatasetFromRetrievalResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x0e\n\x06status\x18\x02 \x01(\t\"E\n\x15GetDatasetDataRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\r\n\x05limit\x18\x03 \x01(\x05\"\x1c\n\nTabularRow\x12\x0e\n\x06values\x18\x01 \x03(\t\"|\n\x16GetDatasetDataResponse\x12\x0f\n\x07\x63olumns\x18\x01 \x03(\t\x12(\n\x04rows\x18\x02 \x03(\x0b\x32\x1a.feast.registry.TabularRow\x12\x12\n\ntotal_rows\x18\x03 \x01(\x05\x12\x13\n\x0bsample_size\x18\x04 \x01(\x05\",\n\x1aGetDatasetJobStatusRequest\x12\x0e\n\x06job_id\x18\x01 \x01(\t\"\x9d\x01\n\x1bGetDatasetJobStatusResponse\x12\x0e\n\x06job_id\x18\x01 \x01(\t\x12\x14\n\x0c\x64\x61taset_name\x18\x02 \x01(\t\x12\x0f\n\x07project\x18\x03 \x01(\t\x12\x0e\n\x06status\x18\x04 \x01(\t\x12\x12\n\ncreated_at\x18\x05 \x01(\t\x12\x14\n\x0c\x63ompleted_at\x18\x06 \x01(\t\x12\r\n\x05\x65rror\x18\x07 \x01(\t\"@\n\x16ListDatasetJobsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x15\n\rstatus_filter\x18\x02 \x01(\t\"T\n\x17ListDatasetJobsResponse\x12\x39\n\x04jobs\x18\x01 \x03(\x0b\x32+.feast.registry.GetDatasetJobStatusResponse\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa3\x02\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\x91\x02\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"~\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\xfa\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x12\x34\n\npagination\x18\x03 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x04 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"u\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"-\n\x0f\x45ntityReference\x12\x0c\n\x04type\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"r\n\x0e\x45ntityRelation\x12/\n\x06source\x18\x01 \x01(\x0b\x32\x1f.feast.registry.EntityReference\x12/\n\x06target\x18\x02 \x01(\x0b\x32\x1f.feast.registry.EntityReference\"\xdf\x01\n\x19GetRegistryLineageRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x1a\n\x12\x66ilter_object_type\x18\x03 \x01(\t\x12\x1a\n\x12\x66ilter_object_name\x18\x04 \x01(\t\x12\x34\n\npagination\x18\x05 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x06 \x01(\x0b\x32\x1d.feast.registry.SortingParams\"\xa8\x02\n\x1aGetRegistryLineageResponse\x12\x35\n\rrelationships\x18\x01 \x03(\x0b\x32\x1e.feast.registry.EntityRelation\x12>\n\x16indirect_relationships\x18\x02 \x03(\x0b\x32\x1e.feast.registry.EntityRelation\x12\x44\n\x18relationships_pagination\x18\x03 \x01(\x0b\x32\".feast.registry.PaginationMetadata\x12M\n!indirect_relationships_pagination\x18\x04 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"\xef\x01\n\x1dGetObjectRelationshipsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0bobject_type\x18\x02 \x01(\t\x12\x13\n\x0bobject_name\x18\x03 \x01(\t\x12\x18\n\x10include_indirect\x18\x04 \x01(\x08\x12\x13\n\x0b\x61llow_cache\x18\x05 \x01(\x08\x12\x34\n\npagination\x18\x06 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x07 \x01(\x0b\x32\x1d.feast.registry.SortingParams\"\x8f\x01\n\x1eGetObjectRelationshipsResponse\x12\x35\n\rrelationships\x18\x01 \x03(\x0b\x32\x1e.feast.registry.EntityRelation\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"\xbe\x02\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\r\n\x05owner\x18\x05 \x01(\t\x12\x35\n\x11\x63reated_timestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x04tags\x18\x08 \x03(\x0b\x32!.feast.registry.Feature.TagsEntry\x12\x0c\n\x04kind\x18\t \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xd3\x01\n\x13ListFeaturesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x06 \x01(\x08\x12\x34\n\npagination\x18\x04 \x01(\x0b\x32 .feast.registry.PaginationParams\x12.\n\x07sorting\x18\x05 \x01(\x0b\x32\x1d.feast.registry.SortingParams\x12\x0c\n\x04kind\x18\x07 \x01(\t\"y\n\x14ListFeaturesResponse\x12)\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x17.feast.registry.Feature\x12\x36\n\npagination\x18\x02 \x01(\x0b\x32\".feast.registry.PaginationMetadata\"]\n\x11GetFeatureRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_view\x18\x02 \x01(\t\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x04 \x01(\x08\x32\xd2(\n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12L\n\x0cGetLabelView\x12#.feast.registry.GetLabelViewRequest\x1a\x15.feast.core.LabelView\"\x00\x12\x61\n\x0eListLabelViews\x12%.feast.registry.ListLabelViewsRequest\x1a&.feast.registry.ListLabelViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x85\x01\n\x1a\x43reateDatasetFromRetrieval\x12\x31.feast.registry.CreateDatasetFromRetrievalRequest\x1a\x32.feast.registry.CreateDatasetFromRetrievalResponse\"\x00\x12\x61\n\x0eGetDatasetData\x12%.feast.registry.GetDatasetDataRequest\x1a&.feast.registry.GetDatasetDataResponse\"\x00\x12p\n\x13GetDatasetJobStatus\x12*.feast.registry.GetDatasetJobStatusRequest\x1a+.feast.registry.GetDatasetJobStatusResponse\"\x00\x12\x64\n\x0fListDatasetJobs\x12&.feast.registry.ListDatasetJobsRequest\x1a\'.feast.registry.ListDatasetJobsResponse\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x12m\n\x12GetRegistryLineage\x12).feast.registry.GetRegistryLineageRequest\x1a*.feast.registry.GetRegistryLineageResponse\"\x00\x12y\n\x16GetObjectRelationships\x12-.feast.registry.GetObjectRelationshipsRequest\x1a..feast.registry.GetObjectRelationshipsResponse\"\x00\x12[\n\x0cListFeatures\x12#.feast.registry.ListFeaturesRequest\x1a$.feast.registry.ListFeaturesResponse\"\x00\x12J\n\nGetFeature\x12!.feast.registry.GetFeatureRequest\x1a\x17.feast.registry.Feature\"\x00\x42\x35Z3github.com/feast-dev/feast/go/protos/feast/registryb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -172,91 +172,91 @@ _globals['_GETSAVEDDATASETREQUEST']._serialized_start=6797 _globals['_GETSAVEDDATASETREQUEST']._serialized_end=6873 _globals['_LISTSAVEDDATASETSREQUEST']._serialized_start=6876 - _globals['_LISTSAVEDDATASETSREQUEST']._serialized_end=7153 + _globals['_LISTSAVEDDATASETSREQUEST']._serialized_end=7192 _globals['_LISTSAVEDDATASETSREQUEST_TAGSENTRY']._serialized_start=1681 _globals['_LISTSAVEDDATASETSREQUEST_TAGSENTRY']._serialized_end=1724 - _globals['_LISTSAVEDDATASETSRESPONSE']._serialized_start=7156 - _globals['_LISTSAVEDDATASETSRESPONSE']._serialized_end=7289 - _globals['_DELETESAVEDDATASETREQUEST']._serialized_start=7291 - _globals['_DELETESAVEDDATASETREQUEST']._serialized_end=7365 - _globals['_CREATEDATASETFROMRETRIEVALREQUEST']._serialized_start=7368 - _globals['_CREATEDATASETFROMRETRIEVALREQUEST']._serialized_end=7832 + _globals['_LISTSAVEDDATASETSRESPONSE']._serialized_start=7195 + _globals['_LISTSAVEDDATASETSRESPONSE']._serialized_end=7328 + _globals['_DELETESAVEDDATASETREQUEST']._serialized_start=7330 + _globals['_DELETESAVEDDATASETREQUEST']._serialized_end=7404 + _globals['_CREATEDATASETFROMRETRIEVALREQUEST']._serialized_start=7407 + _globals['_CREATEDATASETFROMRETRIEVALREQUEST']._serialized_end=7871 _globals['_CREATEDATASETFROMRETRIEVALREQUEST_TAGSENTRY']._serialized_start=1681 _globals['_CREATEDATASETFROMRETRIEVALREQUEST_TAGSENTRY']._serialized_end=1724 - _globals['_CREATEDATASETFROMRETRIEVALRESPONSE']._serialized_start=7834 - _globals['_CREATEDATASETFROMRETRIEVALRESPONSE']._serialized_end=7902 - _globals['_GETDATASETDATAREQUEST']._serialized_start=7904 - _globals['_GETDATASETDATAREQUEST']._serialized_end=7973 - _globals['_TABULARROW']._serialized_start=7975 - _globals['_TABULARROW']._serialized_end=8003 - _globals['_GETDATASETDATARESPONSE']._serialized_start=8005 - _globals['_GETDATASETDATARESPONSE']._serialized_end=8129 - _globals['_GETDATASETJOBSTATUSREQUEST']._serialized_start=8131 - _globals['_GETDATASETJOBSTATUSREQUEST']._serialized_end=8175 - _globals['_GETDATASETJOBSTATUSRESPONSE']._serialized_start=8178 - _globals['_GETDATASETJOBSTATUSRESPONSE']._serialized_end=8335 - _globals['_LISTDATASETJOBSREQUEST']._serialized_start=8337 - _globals['_LISTDATASETJOBSREQUEST']._serialized_end=8401 - _globals['_LISTDATASETJOBSRESPONSE']._serialized_start=8403 - _globals['_LISTDATASETJOBSRESPONSE']._serialized_end=8487 - _globals['_APPLYVALIDATIONREFERENCEREQUEST']._serialized_start=8490 - _globals['_APPLYVALIDATIONREFERENCEREQUEST']._serialized_end=8619 - _globals['_GETVALIDATIONREFERENCEREQUEST']._serialized_start=8621 - _globals['_GETVALIDATIONREFERENCEREQUEST']._serialized_end=8704 - _globals['_LISTVALIDATIONREFERENCESREQUEST']._serialized_start=8707 - _globals['_LISTVALIDATIONREFERENCESREQUEST']._serialized_end=8998 + _globals['_CREATEDATASETFROMRETRIEVALRESPONSE']._serialized_start=7873 + _globals['_CREATEDATASETFROMRETRIEVALRESPONSE']._serialized_end=7941 + _globals['_GETDATASETDATAREQUEST']._serialized_start=7943 + _globals['_GETDATASETDATAREQUEST']._serialized_end=8012 + _globals['_TABULARROW']._serialized_start=8014 + _globals['_TABULARROW']._serialized_end=8042 + _globals['_GETDATASETDATARESPONSE']._serialized_start=8044 + _globals['_GETDATASETDATARESPONSE']._serialized_end=8168 + _globals['_GETDATASETJOBSTATUSREQUEST']._serialized_start=8170 + _globals['_GETDATASETJOBSTATUSREQUEST']._serialized_end=8214 + _globals['_GETDATASETJOBSTATUSRESPONSE']._serialized_start=8217 + _globals['_GETDATASETJOBSTATUSRESPONSE']._serialized_end=8374 + _globals['_LISTDATASETJOBSREQUEST']._serialized_start=8376 + _globals['_LISTDATASETJOBSREQUEST']._serialized_end=8440 + _globals['_LISTDATASETJOBSRESPONSE']._serialized_start=8442 + _globals['_LISTDATASETJOBSRESPONSE']._serialized_end=8526 + _globals['_APPLYVALIDATIONREFERENCEREQUEST']._serialized_start=8529 + _globals['_APPLYVALIDATIONREFERENCEREQUEST']._serialized_end=8658 + _globals['_GETVALIDATIONREFERENCEREQUEST']._serialized_start=8660 + _globals['_GETVALIDATIONREFERENCEREQUEST']._serialized_end=8743 + _globals['_LISTVALIDATIONREFERENCESREQUEST']._serialized_start=8746 + _globals['_LISTVALIDATIONREFERENCESREQUEST']._serialized_end=9037 _globals['_LISTVALIDATIONREFERENCESREQUEST_TAGSENTRY']._serialized_start=1681 _globals['_LISTVALIDATIONREFERENCESREQUEST_TAGSENTRY']._serialized_end=1724 - _globals['_LISTVALIDATIONREFERENCESRESPONSE']._serialized_start=9001 - _globals['_LISTVALIDATIONREFERENCESRESPONSE']._serialized_end=9155 - _globals['_DELETEVALIDATIONREFERENCEREQUEST']._serialized_start=9157 - _globals['_DELETEVALIDATIONREFERENCEREQUEST']._serialized_end=9238 - _globals['_APPLYPERMISSIONREQUEST']._serialized_start=9240 - _globals['_APPLYPERMISSIONREQUEST']._serialized_end=9341 - _globals['_GETPERMISSIONREQUEST']._serialized_start=9343 - _globals['_GETPERMISSIONREQUEST']._serialized_end=9417 - _globals['_LISTPERMISSIONSREQUEST']._serialized_start=9420 - _globals['_LISTPERMISSIONSREQUEST']._serialized_end=9693 + _globals['_LISTVALIDATIONREFERENCESRESPONSE']._serialized_start=9040 + _globals['_LISTVALIDATIONREFERENCESRESPONSE']._serialized_end=9194 + _globals['_DELETEVALIDATIONREFERENCEREQUEST']._serialized_start=9196 + _globals['_DELETEVALIDATIONREFERENCEREQUEST']._serialized_end=9277 + _globals['_APPLYPERMISSIONREQUEST']._serialized_start=9279 + _globals['_APPLYPERMISSIONREQUEST']._serialized_end=9380 + _globals['_GETPERMISSIONREQUEST']._serialized_start=9382 + _globals['_GETPERMISSIONREQUEST']._serialized_end=9456 + _globals['_LISTPERMISSIONSREQUEST']._serialized_start=9459 + _globals['_LISTPERMISSIONSREQUEST']._serialized_end=9732 _globals['_LISTPERMISSIONSREQUEST_TAGSENTRY']._serialized_start=1681 _globals['_LISTPERMISSIONSREQUEST_TAGSENTRY']._serialized_end=1724 - _globals['_LISTPERMISSIONSRESPONSE']._serialized_start=9695 - _globals['_LISTPERMISSIONSRESPONSE']._serialized_end=9821 - _globals['_DELETEPERMISSIONREQUEST']._serialized_start=9823 - _globals['_DELETEPERMISSIONREQUEST']._serialized_end=9895 - _globals['_APPLYPROJECTREQUEST']._serialized_start=9897 - _globals['_APPLYPROJECTREQUEST']._serialized_end=9972 - _globals['_GETPROJECTREQUEST']._serialized_start=9974 - _globals['_GETPROJECTREQUEST']._serialized_end=10028 - _globals['_LISTPROJECTSREQUEST']._serialized_start=10031 - _globals['_LISTPROJECTSREQUEST']._serialized_end=10281 + _globals['_LISTPERMISSIONSRESPONSE']._serialized_start=9734 + _globals['_LISTPERMISSIONSRESPONSE']._serialized_end=9860 + _globals['_DELETEPERMISSIONREQUEST']._serialized_start=9862 + _globals['_DELETEPERMISSIONREQUEST']._serialized_end=9934 + _globals['_APPLYPROJECTREQUEST']._serialized_start=9936 + _globals['_APPLYPROJECTREQUEST']._serialized_end=10011 + _globals['_GETPROJECTREQUEST']._serialized_start=10013 + _globals['_GETPROJECTREQUEST']._serialized_end=10067 + _globals['_LISTPROJECTSREQUEST']._serialized_start=10070 + _globals['_LISTPROJECTSREQUEST']._serialized_end=10320 _globals['_LISTPROJECTSREQUEST_TAGSENTRY']._serialized_start=1681 _globals['_LISTPROJECTSREQUEST_TAGSENTRY']._serialized_end=1724 - _globals['_LISTPROJECTSRESPONSE']._serialized_start=10283 - _globals['_LISTPROJECTSRESPONSE']._serialized_end=10400 - _globals['_DELETEPROJECTREQUEST']._serialized_start=10402 - _globals['_DELETEPROJECTREQUEST']._serialized_end=10454 - _globals['_ENTITYREFERENCE']._serialized_start=10456 - _globals['_ENTITYREFERENCE']._serialized_end=10501 - _globals['_ENTITYRELATION']._serialized_start=10503 - _globals['_ENTITYRELATION']._serialized_end=10617 - _globals['_GETREGISTRYLINEAGEREQUEST']._serialized_start=10620 - _globals['_GETREGISTRYLINEAGEREQUEST']._serialized_end=10843 - _globals['_GETREGISTRYLINEAGERESPONSE']._serialized_start=10846 - _globals['_GETREGISTRYLINEAGERESPONSE']._serialized_end=11142 - _globals['_GETOBJECTRELATIONSHIPSREQUEST']._serialized_start=11145 - _globals['_GETOBJECTRELATIONSHIPSREQUEST']._serialized_end=11384 - _globals['_GETOBJECTRELATIONSHIPSRESPONSE']._serialized_start=11387 - _globals['_GETOBJECTRELATIONSHIPSRESPONSE']._serialized_end=11530 - _globals['_FEATURE']._serialized_start=11533 - _globals['_FEATURE']._serialized_end=11851 + _globals['_LISTPROJECTSRESPONSE']._serialized_start=10322 + _globals['_LISTPROJECTSRESPONSE']._serialized_end=10439 + _globals['_DELETEPROJECTREQUEST']._serialized_start=10441 + _globals['_DELETEPROJECTREQUEST']._serialized_end=10493 + _globals['_ENTITYREFERENCE']._serialized_start=10495 + _globals['_ENTITYREFERENCE']._serialized_end=10540 + _globals['_ENTITYRELATION']._serialized_start=10542 + _globals['_ENTITYRELATION']._serialized_end=10656 + _globals['_GETREGISTRYLINEAGEREQUEST']._serialized_start=10659 + _globals['_GETREGISTRYLINEAGEREQUEST']._serialized_end=10882 + _globals['_GETREGISTRYLINEAGERESPONSE']._serialized_start=10885 + _globals['_GETREGISTRYLINEAGERESPONSE']._serialized_end=11181 + _globals['_GETOBJECTRELATIONSHIPSREQUEST']._serialized_start=11184 + _globals['_GETOBJECTRELATIONSHIPSREQUEST']._serialized_end=11423 + _globals['_GETOBJECTRELATIONSHIPSRESPONSE']._serialized_start=11426 + _globals['_GETOBJECTRELATIONSHIPSRESPONSE']._serialized_end=11569 + _globals['_FEATURE']._serialized_start=11572 + _globals['_FEATURE']._serialized_end=11890 _globals['_FEATURE_TAGSENTRY']._serialized_start=1681 _globals['_FEATURE_TAGSENTRY']._serialized_end=1724 - _globals['_LISTFEATURESREQUEST']._serialized_start=11854 - _globals['_LISTFEATURESREQUEST']._serialized_end=12065 - _globals['_LISTFEATURESRESPONSE']._serialized_start=12067 - _globals['_LISTFEATURESRESPONSE']._serialized_end=12188 - _globals['_GETFEATUREREQUEST']._serialized_start=12190 - _globals['_GETFEATUREREQUEST']._serialized_end=12283 - _globals['_REGISTRYSERVER']._serialized_start=12286 - _globals['_REGISTRYSERVER']._serialized_end=17488 + _globals['_LISTFEATURESREQUEST']._serialized_start=11893 + _globals['_LISTFEATURESREQUEST']._serialized_end=12104 + _globals['_LISTFEATURESRESPONSE']._serialized_start=12106 + _globals['_LISTFEATURESRESPONSE']._serialized_end=12227 + _globals['_GETFEATUREREQUEST']._serialized_start=12229 + _globals['_GETFEATUREREQUEST']._serialized_end=12322 + _globals['_REGISTRYSERVER']._serialized_start=12325 + _globals['_REGISTRYSERVER']._serialized_end=17527 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.pyi b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.pyi index 9171c75a5be..e76775d0004 100644 --- a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.pyi +++ b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.pyi @@ -1218,6 +1218,8 @@ class ListSavedDatasetsRequest(google.protobuf.message.Message): TAGS_FIELD_NUMBER: builtins.int PAGINATION_FIELD_NUMBER: builtins.int SORTING_FIELD_NUMBER: builtins.int + NAMESPACE_FIELD_NUMBER: builtins.int + COLLECTION_FIELD_NUMBER: builtins.int project: builtins.str allow_cache: builtins.bool @property @@ -1226,6 +1228,10 @@ class ListSavedDatasetsRequest(google.protobuf.message.Message): def pagination(self) -> global___PaginationParams: ... @property def sorting(self) -> global___SortingParams: ... + namespace: builtins.str + """Optional logical namespace filter. Empty string means no filter.""" + collection: builtins.str + """Optional collection filter. Empty string means no filter.""" def __init__( self, *, @@ -1234,9 +1240,11 @@ class ListSavedDatasetsRequest(google.protobuf.message.Message): tags: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., pagination: global___PaginationParams | None = ..., sorting: global___SortingParams | None = ..., + namespace: builtins.str = ..., + collection: builtins.str = ..., ) -> None: ... def HasField(self, field_name: typing_extensions.Literal["pagination", b"pagination", "sorting", b"sorting"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["allow_cache", b"allow_cache", "pagination", b"pagination", "project", b"project", "sorting", b"sorting", "tags", b"tags"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["allow_cache", b"allow_cache", "collection", b"collection", "namespace", b"namespace", "pagination", b"pagination", "project", b"project", "sorting", b"sorting", "tags", b"tags"]) -> None: ... global___ListSavedDatasetsRequest = ListSavedDatasetsRequest diff --git a/sdk/python/feast/registry_server.py b/sdk/python/feast/registry_server.py index 8c6a979cc8a..f04bdafa194 100644 --- a/sdk/python/feast/registry_server.py +++ b/sdk/python/feast/registry_server.py @@ -940,6 +940,8 @@ def GetSavedDataset( def ListSavedDatasets( self, request: RegistryServer_pb2.ListSavedDatasetsRequest, context ): + namespace_filter = request.namespace if request.namespace else None + collection_filter = request.collection if request.collection else None paginated_saved_datasets, pagination_metadata = apply_pagination_and_sorting( permitted_resources( resources=cast( @@ -948,6 +950,8 @@ def ListSavedDatasets( project=request.project, allow_cache=request.allow_cache, tags=dict(request.tags), + namespace=namespace_filter, + collection=collection_filter, ), ), actions=AuthzedAction.DESCRIBE, diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index de83a2db163..b6ee3885b3c 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -337,6 +337,39 @@ def to_openlineage_config(self): ) +class EmbeddingModelConfig(FeastConfigBaseModel): + """Configuration for the query-time embedding model used by the feature server. + + Required when using ``openai_search`` or the + ``/v1/vector_stores/{vector_store_id}/search`` endpoint. + + **Sentence Transformers** (default) — runs locally, no API key required. + Ideal for air-gapped or cost-sensitive deployments. Requires the + ``sentence-transformers`` package (``pip install sentence-transformers``). + + Example in ``feature_store.yaml``:: + + embedding_model: + provider: sentence_transformers # default; can be omitted + model: all-MiniLM-L6-v2 + + Custom providers can be plugged in by implementing the + :class:`~feast.embedder.EmbeddingProvider` protocol and passing an + instance to :class:`~feast.feature_store.FeatureStore`. + """ + + provider: str = "sentence_transformers" + """Embedding backend to use. Supported values: + ``'sentence_transformers'`` (default).""" + + model: str + """Model identifier. + + Any HuggingFace model name compatible with ``SentenceTransformer``, + e.g. ``'all-MiniLM-L6-v2'``, ``'BAAI/bge-small-en-v1.5'``. + """ + + class RepoConfig(FeastBaseModel): """Repo config. Typically loaded from `feature_store.yaml`""" @@ -376,6 +409,13 @@ class RepoConfig(FeastBaseModel): feature_server: Optional[Any] = None """ FeatureServerConfig: Feature server configuration (optional depending on provider) """ + embedding_model: Optional[EmbeddingModelConfig] = Field( + None, alias="embedding_model" + ) + """ EmbeddingModelConfig: Embedding model configuration. + Required when using openai_search or the + OpenAI-compatible vector store search endpoint. """ + flags: Any = None """ Flags (deprecated field): Feature flags for experimental features """ diff --git a/sdk/python/feast/saved_dataset.py b/sdk/python/feast/saved_dataset.py index 9ed69861d4b..d78c5b4c349 100644 --- a/sdk/python/feast/saved_dataset.py +++ b/sdk/python/feast/saved_dataset.py @@ -83,6 +83,9 @@ class SavedDataset: storage: SavedDatasetStorage tags: Dict[str, str] feature_service_name: Optional[str] = None + namespace: str = "" + collection: str = "" + description: str = "" created_timestamp: Optional[datetime] = None last_updated_timestamp: Optional[datetime] = None @@ -101,6 +104,9 @@ def __init__( full_feature_names: bool = False, tags: Optional[Dict[str, str]] = None, feature_service_name: Optional[str] = None, + namespace: str = "", + collection: str = "", + description: str = "", ): self.name = name self.features = features @@ -109,6 +115,9 @@ def __init__( self.full_feature_names = full_feature_names self.tags = tags or {} self.feature_service_name = feature_service_name + self.namespace = namespace + self.collection = collection + self.description = description self._retrieval_job = None @@ -136,6 +145,9 @@ def __eq__(self, other): or self.full_feature_names != other.full_feature_names or self.tags != other.tags or self.feature_service_name != other.feature_service_name + or self.namespace != other.namespace + or self.collection != other.collection + or self.description != other.description ): return False @@ -156,6 +168,9 @@ def from_proto(saved_dataset_proto: SavedDatasetProto): full_feature_names=saved_dataset_proto.spec.full_feature_names, storage=SavedDatasetStorage.from_proto(saved_dataset_proto.spec.storage), tags=dict(saved_dataset_proto.spec.tags.items()), + namespace=saved_dataset_proto.spec.namespace, + collection=saved_dataset_proto.spec.collection, + description=saved_dataset_proto.spec.description, ) if saved_dataset_proto.spec.feature_service_name: @@ -202,6 +217,9 @@ def to_proto(self) -> SavedDatasetProto: full_feature_names=self.full_feature_names, storage=self.storage.to_proto(), tags=self.tags, + namespace=self.namespace, + collection=self.collection, + description=self.description, ) if self.feature_service_name: spec.feature_service_name = self.feature_service_name diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 4c10c1903e5..0d0f15ff4a4 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1779,3 +1779,24 @@ def _get_feature_view_vector_field_metadata( if not vector_fields: return None return vector_fields[0] + + +def _distance_to_score(distance: float, metric: Optional[str] = None) -> float: + """Convert a raw distance value into a higher-is-better relevance score. + + OpenAI clients treat ``score`` as higher-is-better (used with + ``score_threshold``). Online stores return raw *distance* values where + lower is better, so a metric-aware conversion is required. + + * **L2 / euclidean**: ``1 / (1 + distance)`` — maps [0, inf) to (0, 1] + * **cosine** (pgvector ``<=>`` returns ``1 - cosine_similarity``): + ``1 - distance`` — recovers cosine similarity, clamped >= 0 + * **inner_product / ip / dot** (pgvector ``<#>`` returns ``-dot(a, b)``): + ``-distance`` — recovers the raw inner product + """ + m = (metric or "L2").lower() + if m in ("cosine",): + return max(0.0, 1.0 - distance) + if m in ("inner_product", "ip", "dot"): + return -distance + return 1.0 / (1.0 + distance) diff --git a/sdk/python/feast/vector_store_utils.py b/sdk/python/feast/vector_store_utils.py new file mode 100644 index 00000000000..e04a7bb4936 --- /dev/null +++ b/sdk/python/feast/vector_store_utils.py @@ -0,0 +1,66 @@ +import hashlib +from typing import Any, Dict, List, Optional + +from feast.errors import FeatureViewNotFoundException +from feast.feature_view import FeatureView + + +def feature_view_to_vs_id(project: str, feature_view_name: str) -> str: + digest = hashlib.sha256(f"{project}:{feature_view_name}".encode()).hexdigest()[:24] + return f"vs_{digest}" + + +def _has_vector_index(fv: FeatureView) -> bool: + return any(getattr(f, "vector_index", False) for f in fv.schema) + + +def build_vector_store_object(project: str, fv: FeatureView) -> Dict[str, Any]: + return { + "id": feature_view_to_vs_id(project, fv.name), + "object": "vector_store", + "name": fv.name, + "status": "completed", + "created_at": int(fv.created_timestamp.timestamp()) + if fv.created_timestamp + else 0, + } + + +class VectorStoreRegistry: + """Cached mapping from vs_{hash} IDs to FeatureViews. + + Built once at server startup, refreshed on registry TTL cycle. + """ + + def __init__(self, store: Any): + self._store = store + self._id_to_fv: Dict[str, FeatureView] = {} + self._vector_store_objects: List[Dict[str, Any]] = [] + self.refresh() + + def refresh(self) -> None: + project = self._store.project + id_to_fv: Dict[str, FeatureView] = {} + objects: List[Dict[str, Any]] = [] + for fv in self._store.list_feature_views(): + if _has_vector_index(fv): + vs_id = feature_view_to_vs_id(project, fv.name) + id_to_fv[vs_id] = fv + objects.append(build_vector_store_object(project, fv)) + self._id_to_fv = id_to_fv + self._vector_store_objects = objects + + def resolve(self, vs_id: str) -> FeatureView: + fv = self._id_to_fv.get(vs_id) + if fv is None: + raise FeatureViewNotFoundException(vs_id) + return fv + + def list_vector_stores(self) -> List[Dict[str, Any]]: + return self._vector_store_objects + + def get_vector_store(self, vs_id: str) -> Optional[Dict[str, Any]]: + fv = self._id_to_fv.get(vs_id) + if fv is None: + return None + return build_vector_store_object(self._store.project, fv) diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 82cfc7fb513..2586db80353 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -19,6 +19,7 @@ from feast import Entity, FeatureService, FeatureView, RepoConfig from feast.data_source import DataSource +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.infra.offline_stores.offline_store import RetrievalJob from feast.infra.provider import Provider from feast.infra.registry.base_registry import BaseRegistry @@ -171,10 +172,11 @@ def retrieve_online_documents_v2( config: RepoConfig, table: FeatureView, requested_features: List[str], - query: Optional[List[float]], + embedding: Optional[List[float]], top_k: int, distance_metric: Optional[str] = None, query_string: Optional[str] = None, + filters: Optional[Union[ComparisonFilter, CompoundFilter]] = None, include_feature_view_version_metadata: bool = False, ) -> List[ Tuple[ diff --git a/sdk/python/tests/integration/online_store/test_universal_online.py b/sdk/python/tests/integration/online_store/test_universal_online.py index 109eea454c0..41b0e6a635f 100644 --- a/sdk/python/tests/integration/online_store/test_universal_online.py +++ b/sdk/python/tests/integration/online_store/test_universal_online.py @@ -1,3 +1,4 @@ +import asyncio import os import random import time @@ -14,10 +15,11 @@ from feast import FeatureStore from feast.entity import Entity -from feast.errors import FeatureNameCollisionError +from feast.errors import FeatureNameCollisionError, FeatureViewNotFoundException from feast.feature_service import FeatureService from feast.feature_view import FeatureView from feast.field import Field +from feast.filter_models import ComparisonFilter, CompoundFilter from feast.infra.offline_stores.file_source import FileSource from feast.infra.utils.postgres.postgres_config import ConnectionType from feast.online_response import TIMESTAMP_POSTFIX @@ -31,6 +33,7 @@ ValueType, ) from feast.utils import _utc_now +from feast.vector_store_utils import feature_view_to_vs_id from feast.wait import wait_retry_backoff from tests.universal.feature_repos.repo_configuration import ( Environment, @@ -1332,3 +1335,305 @@ def test_retrieve_online_documents_v2(environment, fake_document_data): assert len(no_match_results["text_field"]) == 0 assert "text_rank" in no_match_results assert len(no_match_results["text_rank"]) == 0 + + +def _setup_documents_with_categories(fs): + """Shared helper that creates and populates a feature view with embeddings, + text, and category fields. Returns (feature_view, entity, dataframe).""" + n_rows = 20 + vector_dim = 2 + random.seed(42) + + df = pd.DataFrame( + { + "item_id": list(range(n_rows)), + "chunk_id": list(range(n_rows)), + "embedding": [list(np.random.random(vector_dim)) for _ in range(n_rows)], + "text_field": [ + f"Document text content {i} with searchable keywords" + for i in range(n_rows) + ], + "category": [f"Category-{i % 5}" for i in range(n_rows)], + "event_timestamp": [datetime.now() for _ in range(n_rows)], + } + ) + + data_source = FileSource( + path="dummy_path.parquet", timestamp_field="event_timestamp" + ) + + item_entity = Entity( + name="item_id", + join_keys=["item_id"], + value_type=ValueType.INT64, + ) + + item_embeddings_fv = FeatureView( + name="item_embeddings", + entities=[item_entity], + schema=[ + Field(name="embedding", dtype=Array(Float32), vector_index=True), + Field(name="text_field", dtype=String), + Field(name="category", dtype=String), + Field(name="item_id", dtype=Int64), + Field(name="chunk_id", dtype=Int64), + ], + source=data_source, + ) + + fs.apply([item_embeddings_fv, item_entity]) + fs.write_to_online_store("item_embeddings", df) + return item_embeddings_fv, item_entity, df + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch"]) +def test_retrieve_online_documents_v2_with_filters(environment, fake_document_data): + """Test that metadata filters narrow down vector/text search results.""" + fs = environment.feature_store + fs.config.online_store.vector_enabled = True + + _, _, df = _setup_documents_with_categories(fs) + vector_dim = 2 + query_embedding = list(np.random.random(vector_dim)) + + # --- eq filter: only Category-0 rows --- + eq_filter = ComparisonFilter(type="eq", key="category", value="Category-0") + results = fs.retrieve_online_documents_v2( + features=[ + "item_embeddings:embedding", + "item_embeddings:text_field", + "item_embeddings:category", + "item_embeddings:item_id", + ], + query=query_embedding, + top_k=10, + distance_metric="L2", + filters=eq_filter, + ).to_dict() + + assert len(results["category"]) > 0 + assert len(results["category"]) <= 4 # 20 rows / 5 categories + assert all(c == "Category-0" for c in results["category"]) + + # --- ne filter: exclude Category-0 --- + ne_filter = ComparisonFilter(type="ne", key="category", value="Category-0") + results = fs.retrieve_online_documents_v2( + features=[ + "item_embeddings:embedding", + "item_embeddings:text_field", + "item_embeddings:category", + "item_embeddings:item_id", + ], + query=query_embedding, + top_k=10, + distance_metric="L2", + filters=ne_filter, + ).to_dict() + + assert len(results["category"]) > 0 + assert all(c != "Category-0" for c in results["category"]) + + # --- in filter: Category-0 or Category-1 --- + in_filter = ComparisonFilter( + type="in", key="category", value=["Category-0", "Category-1"] + ) + results = fs.retrieve_online_documents_v2( + features=[ + "item_embeddings:embedding", + "item_embeddings:text_field", + "item_embeddings:category", + "item_embeddings:item_id", + ], + query=query_embedding, + top_k=10, + distance_metric="L2", + filters=in_filter, + ).to_dict() + + assert len(results["category"]) > 0 + assert all(c in ("Category-0", "Category-1") for c in results["category"]) + + # --- compound AND filter: category == Category-0 AND chunk_id >= 5 --- + and_filter = CompoundFilter( + type="and", + filters=[ + ComparisonFilter(type="eq", key="category", value="Category-0"), + ComparisonFilter(type="gte", key="chunk_id", value=5), + ], + ) + results = fs.retrieve_online_documents_v2( + features=[ + "item_embeddings:embedding", + "item_embeddings:text_field", + "item_embeddings:category", + "item_embeddings:chunk_id", + ], + query=query_embedding, + top_k=10, + distance_metric="L2", + filters=and_filter, + ).to_dict() + + assert len(results["category"]) > 0 + assert all(c == "Category-0" for c in results["category"]) + assert all(i >= 5 for i in results["chunk_id"]) + + # --- text search + filter --- + text_filter = ComparisonFilter(type="eq", key="category", value="Category-2") + text_results = fs.retrieve_online_documents_v2( + features=[ + "item_embeddings:embedding", + "item_embeddings:text_field", + "item_embeddings:category", + "item_embeddings:item_id", + ], + query_string="searchable keywords", + top_k=10, + filters=text_filter, + ).to_dict() + + assert len(text_results["category"]) > 0 + assert all(c == "Category-2" for c in text_results["category"]) + + # --- filter with no matches --- + empty_filter = ComparisonFilter( + type="eq", key="category", value="NonexistentCategory" + ) + empty_results = fs.retrieve_online_documents_v2( + features=[ + "item_embeddings:embedding", + "item_embeddings:text_field", + "item_embeddings:category", + "item_embeddings:item_id", + ], + query=query_embedding, + top_k=10, + distance_metric="L2", + filters=empty_filter, + ).to_dict() + + assert len(empty_results.get("category", [])) == 0 + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch"]) +def test_openai_search(environment, fake_document_data): + """Test OpenAI-compatible vector store search returns the correct response shape.""" + fs = environment.feature_store + fs.config.online_store.vector_enabled = True + + fv, _, df = _setup_documents_with_categories(fs) + vector_dim = 2 + vs_id = feature_view_to_vs_id(fs.project, "item_embeddings") + + fake_embedding = list(np.random.random(vector_dim)) + + mock_provider = unittest.mock.MagicMock() + mock_provider.aembed = unittest.mock.AsyncMock(return_value=[fake_embedding]) + fs.embedding_provider = mock_provider + + result = asyncio.run( + fs.openai_search( + vector_store_id="item_embeddings", + query="test query", + max_num_results=5, + ) + ) + + # Validate top-level OpenAI response shape + assert result["object"] == "vector_store.search_results.page" + assert isinstance(result["search_query"], list) + assert result["search_query"] == ["test query"] + assert result["has_more"] is False + assert result["next_page"] is None + + assert isinstance(result["data"], list) + assert len(result["data"]) > 0 + assert len(result["data"]) <= 5 + + seen_file_ids = set() + for item_result in result["data"]: + assert "file_id" in item_result + fid = item_result["file_id"] + assert fid.startswith(f"{vs_id}_") + seen_file_ids.add(fid) + assert "filename" in item_result + assert item_result["filename"] == vs_id + assert "score" in item_result + assert isinstance(item_result["score"], float) + assert "attributes" in item_result + assert isinstance(item_result["attributes"], dict) + assert "item_id" not in item_result["attributes"] + assert "content" in item_result + assert isinstance(item_result["content"], list) + for part in item_result["content"]: + assert "type" in part + assert part["type"] == "text" + assert "text" in part + assert len(seen_file_ids) == len(result["data"]) + + # --- Test with features_to_retrieve --- + result_subset = asyncio.run( + fs.openai_search( + vector_store_id="item_embeddings", + query="test query", + max_num_results=5, + features_to_retrieve=["text_field", "category"], + ) + ) + + assert len(result_subset["data"]) > 0 + for item_result in result_subset["data"]: + attr_keys = set(item_result["attributes"].keys()) + assert "embedding" not in attr_keys + + # --- Test with list query --- + result_list = asyncio.run( + fs.openai_search( + vector_store_id="item_embeddings", + query=["term1", "term2"], + max_num_results=5, + ) + ) + + assert result_list["search_query"] == ["term1", "term2"] + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch"]) +def test_openai_search_no_embedding_config(environment, fake_document_data): + """Test that openai_search raises ValueError + when no embedding provider is set and embedding_model is not configured.""" + fs = environment.feature_store + fs.config.online_store.vector_enabled = True + _setup_documents_with_categories(fs) + fs.config.embedding_model = None + fs._embedding_provider = None + + with pytest.raises(ValueError, match="No embedding provider set"): + asyncio.run( + fs.openai_search( + vector_store_id="item_embeddings", + query="test query", + ) + ) + + +@pytest.mark.integration +@pytest.mark.universal_online_stores(only=["pgvector", "elasticsearch"]) +def test_openai_search_not_found(environment, fake_document_data): + """Test that openai_search raises FeatureViewNotFoundException + for a non-existent feature view.""" + fs = environment.feature_store + mock_provider = unittest.mock.MagicMock() + mock_provider.aembed = unittest.mock.AsyncMock(return_value=[[0.0, 0.0]]) + fs.embedding_provider = mock_provider + + with pytest.raises(FeatureViewNotFoundException): + asyncio.run( + fs.openai_search( + vector_store_id="nonexistent_feature_view", + query="test query", + ) + ) diff --git a/sdk/python/tests/unit/api/test_api_rest_registry_server.py b/sdk/python/tests/unit/api/test_api_rest_registry_server.py index 34e119d1b53..464e033a27e 100644 --- a/sdk/python/tests/unit/api/test_api_rest_registry_server.py +++ b/sdk/python/tests/unit/api/test_api_rest_registry_server.py @@ -1,4 +1,4 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import pytest @@ -11,23 +11,36 @@ def mock_store_and_registry(): mock_registry = MagicMock() mock_store = MagicMock(spec=FeatureStore) mock_store.registry = mock_registry - mock_store.config = MagicMock() + mock_store.project = "test_project" + mock_config = MagicMock() + mock_store.config = mock_config + mock_config.auth_config.type = "no_auth" + # MagicMock makes nested attrs truthy; keep MCP off unless a test opts in. + mock_config.registry.mcp = None return mock_store, mock_registry -@patch("feast.api.registry.rest.rest_registry_server.RegistryServer") -@patch("feast.api.registry.rest.rest_registry_server.register_all_routes") -@patch("feast.api.registry.rest.rest_registry_server.init_security_manager") -@patch("feast.api.registry.rest.rest_registry_server.init_auth_manager") -@patch("feast.api.registry.rest.rest_registry_server.get_auth_manager") +@pytest.mark.xdist_group(name="rest_registry_server") def test_rest_registry_server_initializes_correctly( - mock_get_auth_manager, - mock_init_auth_manager, - mock_init_security_manager, - mock_register_all_routes, - mock_registry_server_cls, mock_store_and_registry, + mocker, ): + mock_get_auth_manager = mocker.patch( + "feast.api.registry.rest.rest_registry_server.get_auth_manager" + ) + mock_init_auth_manager = mocker.patch( + "feast.api.registry.rest.rest_registry_server.init_auth_manager" + ) + mock_init_security_manager = mocker.patch( + "feast.api.registry.rest.rest_registry_server.init_security_manager" + ) + mock_register_all_routes = mocker.patch( + "feast.api.registry.rest.rest_registry_server.register_all_routes" + ) + mock_registry_server_cls = mocker.patch( + "feast.api.registry.rest.rest_registry_server.RegistryServer" + ) + store, registry = mock_store_and_registry mock_grpc_handler = MagicMock() mock_registry_server_cls.return_value = mock_grpc_handler diff --git a/sdk/python/tests/unit/infra/feature_servers/test_mcp_server.py b/sdk/python/tests/unit/infra/feature_servers/test_mcp_server.py index 7fe0fa3d99a..e8809f84577 100644 --- a/sdk/python/tests/unit/infra/feature_servers/test_mcp_server.py +++ b/sdk/python/tests/unit/infra/feature_servers/test_mcp_server.py @@ -6,6 +6,136 @@ from feast.feature_store import FeatureStore from feast.infra.mcp_servers.mcp_config import McpFeatureServerConfig +from feast.infra.mcp_servers.mcp_server import _resolve_schema_references_safe + + +class TestResolveSchemaReferencesSafe(unittest.TestCase): + """Tests for the circular-ref-safe OpenAPI schema resolver.""" + + def test_simple_ref_resolution(self): + reference_schema = { + "components": { + "schemas": { + "Pet": { + "type": "object", + "properties": {"name": {"type": "string"}}, + } + } + } + } + schema_part = {"$ref": "#/components/schemas/Pet"} + result = _resolve_schema_references_safe(schema_part, reference_schema) + self.assertEqual(result["type"], "object") + self.assertNotIn("$ref", result) + self.assertIn("name", result["properties"]) + + def test_circular_ref_breaks_cycle(self): + """Value -> Struct -> Value must not recurse infinitely.""" + reference_schema = { + "components": { + "schemas": { + "Value": { + "type": "object", + "properties": { + "struct_value": {"$ref": "#/components/schemas/Struct"}, + }, + }, + "Struct": { + "type": "object", + "properties": { + "fields": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/Value" + }, + } + }, + }, + } + } + } + schema_part = {"$ref": "#/components/schemas/Value"} + result = _resolve_schema_references_safe(schema_part, reference_schema) + self.assertEqual(result["type"], "object") + self.assertNotIn("$ref", result) + # The circular Value ref should be replaced with {"type": "object"} + additional = result["properties"]["struct_value"]["properties"]["fields"][ + "additionalProperties" + ] + self.assertEqual(additional.get("type"), "object") + self.assertNotIn("$ref", additional) + + def test_self_referential_schema(self): + """A schema referencing itself (e.g. a tree node).""" + reference_schema = { + "components": { + "schemas": { + "TreeNode": { + "type": "object", + "properties": { + "children": { + "type": "array", + "items": {"$ref": "#/components/schemas/TreeNode"}, + } + }, + } + } + } + } + schema_part = {"$ref": "#/components/schemas/TreeNode"} + result = _resolve_schema_references_safe(schema_part, reference_schema) + self.assertEqual(result["type"], "object") + child_items = result["properties"]["children"]["items"] + self.assertEqual(child_items.get("type"), "object") + self.assertNotIn("$ref", child_items) + + def test_no_ref_passthrough(self): + schema_part = {"type": "string", "description": "a name"} + result = _resolve_schema_references_safe(schema_part, {}) + self.assertEqual(result, schema_part) + + def test_unknown_ref_preserved(self): + schema_part = {"$ref": "#/components/schemas/Missing"} + reference_schema = {"components": {"schemas": {}}} + result = _resolve_schema_references_safe(schema_part, reference_schema) + self.assertIn("$ref", result) + + def test_does_not_mutate_input(self): + reference_schema = { + "components": { + "schemas": { + "Foo": {"type": "object"}, + } + } + } + schema_part = {"$ref": "#/components/schemas/Foo"} + original_copy = schema_part.copy() + _resolve_schema_references_safe(schema_part, reference_schema) + self.assertEqual(schema_part, original_copy) + + +class TestPatchFastapiMcpSchemaResolver(unittest.TestCase): + """Tests for the monkey-patch helper.""" + + @patch("feast.infra.mcp_servers.mcp_server.MCP_AVAILABLE", True) + def test_patch_replaces_function(self): + from feast.infra.mcp_servers.mcp_server import ( + _patch_fastapi_mcp_schema_resolver, + _resolve_schema_references_safe, + ) + + try: + import fastapi_mcp.openapi.utils as mcp_utils + + original = mcp_utils.resolve_schema_references + _patch_fastapi_mcp_schema_resolver() + self.assertIs( + mcp_utils.resolve_schema_references, + _resolve_schema_references_safe, + ) + mcp_utils.resolve_schema_references = original + except ImportError: + self.skipTest("fastapi_mcp not installed") class TestMcpFeatureServerConfig(unittest.TestCase): @@ -280,13 +410,13 @@ def test_mcp_mounted_when_enabled( mock_store.registry = Mock() mock_store.project = "test_project" - mock_mcp_instance = Mock() + mock_mcp_instance = Mock(spec_set=["mount_sse", "mount", "mount_http"]) mock_fast_api_mcp.return_value = mock_mcp_instance server = RestRegistryServer(mock_store) mock_fast_api_mcp.assert_called_once_with(server.app, name="feast-registry-mcp") - mock_mcp_instance.mount.assert_called_once() + mock_mcp_instance.mount_sse.assert_called_once() @patch("feast.api.registry.rest.rest_registry_server.RestRegistryServer._init_auth") @patch( diff --git a/sdk/python/tests/unit/test_filter_models.py b/sdk/python/tests/unit/test_filter_models.py new file mode 100644 index 00000000000..d5db20a1330 --- /dev/null +++ b/sdk/python/tests/unit/test_filter_models.py @@ -0,0 +1,160 @@ +import pytest +from pydantic import ValidationError + +from feast.filter_models import ( + ComparisonFilter, + CompoundFilter, + convert_dict_to_filter, +) + + +class TestComparisonFilter: + @pytest.mark.parametrize("op", ["eq", "ne", "gt", "gte", "lt", "lte", "in", "nin"]) + def test_valid_operators(self, op): + f = ComparisonFilter(type=op, key="field", value="x") + assert f.type == op + assert f.key == "field" + + def test_rejects_invalid_operator(self): + with pytest.raises(ValidationError): + ComparisonFilter(type="like", key="field", value="x") + + def test_accepts_string_value(self): + f = ComparisonFilter(type="eq", key="city", value="LA") + assert f.value == "LA" + + def test_accepts_int_value(self): + f = ComparisonFilter(type="gt", key="age", value=25) + assert f.value == 25 + + def test_accepts_float_value(self): + f = ComparisonFilter(type="lte", key="score", value=0.95) + assert f.value == 0.95 + + def test_accepts_bool_value(self): + f = ComparisonFilter(type="eq", key="active", value=True) + assert f.value is True + + def test_accepts_list_value(self): + f = ComparisonFilter(type="in", key="status", value=["a", "b"]) + assert f.value == ["a", "b"] + + +class TestCompoundFilter: + def test_and_filter(self): + f = CompoundFilter( + type="and", + filters=[ + ComparisonFilter(type="eq", key="a", value=1), + ComparisonFilter(type="gt", key="b", value=2), + ], + ) + assert f.type == "and" + assert len(f.filters) == 2 + + def test_or_filter(self): + f = CompoundFilter( + type="or", + filters=[ + ComparisonFilter(type="eq", key="a", value=1), + ComparisonFilter(type="eq", key="b", value=2), + ], + ) + assert f.type == "or" + assert len(f.filters) == 2 + + def test_rejects_invalid_type(self): + with pytest.raises(ValidationError): + CompoundFilter( + type="xor", + filters=[ComparisonFilter(type="eq", key="a", value=1)], + ) + + def test_nested_compound(self): + f = CompoundFilter( + type="and", + filters=[ + ComparisonFilter(type="eq", key="x", value=1), + CompoundFilter( + type="or", + filters=[ + ComparisonFilter(type="gt", key="y", value=5), + ComparisonFilter(type="lt", key="z", value=10), + ], + ), + ], + ) + assert f.type == "and" + assert len(f.filters) == 2 + inner = f.filters[1] + assert isinstance(inner, CompoundFilter) + assert inner.type == "or" + assert len(inner.filters) == 2 + + +class TestConvertDictToFilter: + def test_comparison_dict(self): + result = convert_dict_to_filter({"type": "eq", "key": "city", "value": "LA"}) + assert isinstance(result, ComparisonFilter) + assert result.type == "eq" + assert result.key == "city" + assert result.value == "LA" + + def test_compound_dict(self): + result = convert_dict_to_filter( + { + "type": "and", + "filters": [ + {"type": "eq", "key": "a", "value": 1}, + {"type": "gt", "key": "b", "value": 2}, + ], + } + ) + assert isinstance(result, CompoundFilter) + assert result.type == "and" + assert len(result.filters) == 2 + assert all(isinstance(f, ComparisonFilter) for f in result.filters) + + def test_nested_compound_dict(self): + result = convert_dict_to_filter( + { + "type": "or", + "filters": [ + {"type": "eq", "key": "x", "value": "a"}, + { + "type": "and", + "filters": [ + {"type": "gt", "key": "y", "value": 5}, + {"type": "lt", "key": "z", "value": 10}, + ], + }, + ], + } + ) + assert isinstance(result, CompoundFilter) + assert result.type == "or" + inner = result.filters[1] + assert isinstance(inner, CompoundFilter) + assert inner.type == "and" + assert len(inner.filters) == 2 + + def test_or_compound_dict(self): + result = convert_dict_to_filter( + { + "type": "or", + "filters": [ + {"type": "eq", "key": "a", "value": 1}, + {"type": "eq", "key": "b", "value": 2}, + ], + } + ) + assert isinstance(result, CompoundFilter) + assert result.type == "or" + + def test_in_operator_dict(self): + result = convert_dict_to_filter( + {"type": "in", "key": "status", "value": ["active", "pending"]} + ) + assert isinstance(result, ComparisonFilter) + assert result.type == "in" + assert result.value == ["active", "pending"] diff --git a/sdk/python/tests/unit/test_vector_store_utils.py b/sdk/python/tests/unit/test_vector_store_utils.py new file mode 100644 index 00000000000..135519ba8f1 --- /dev/null +++ b/sdk/python/tests/unit/test_vector_store_utils.py @@ -0,0 +1,123 @@ +import unittest.mock +from datetime import datetime, timezone + +import pytest + +from feast.errors import FeatureViewNotFoundException +from feast.feature_view import FeatureView +from feast.field import Field +from feast.infra.offline_stores.file_source import FileSource +from feast.types import Array, Float32, String +from feast.vector_store_utils import ( + VectorStoreRegistry, + build_vector_store_object, + feature_view_to_vs_id, +) + +_DUMMY_SOURCE = FileSource(path="dummy.parquet", timestamp_field="ts") + + +def _make_fv(name: str, vector_index: bool = True) -> FeatureView: + schema = [ + Field(name="embedding", dtype=Array(Float32), vector_index=vector_index), + Field(name="text", dtype=String), + ] + fv = FeatureView(name=name, schema=schema, source=_DUMMY_SOURCE) + fv.created_timestamp = datetime(2025, 1, 1, tzinfo=timezone.utc) + return fv + + +class TestFeatureViewToVsId: + def test_deterministic(self): + id1 = feature_view_to_vs_id("proj", "fv_name") + id2 = feature_view_to_vs_id("proj", "fv_name") + assert id1 == id2 + + def test_format(self): + vs_id = feature_view_to_vs_id("proj", "fv_name") + assert vs_id.startswith("vs_") + assert len(vs_id) == 27 # "vs_" + 24 hex chars + + def test_different_projects_produce_different_ids(self): + id_a = feature_view_to_vs_id("project_a", "embeddings") + id_b = feature_view_to_vs_id("project_b", "embeddings") + assert id_a != id_b + + def test_different_fv_names_produce_different_ids(self): + id_a = feature_view_to_vs_id("proj", "fv_one") + id_b = feature_view_to_vs_id("proj", "fv_two") + assert id_a != id_b + + +class TestBuildVectorStoreObject: + def test_shape(self): + fv = _make_fv("my_embeddings") + obj = build_vector_store_object("proj", fv) + assert obj["object"] == "vector_store" + assert obj["name"] == "my_embeddings" + assert obj["status"] == "completed" + assert obj["id"] == feature_view_to_vs_id("proj", "my_embeddings") + assert isinstance(obj["created_at"], int) + assert obj["created_at"] > 0 + + def test_no_created_timestamp(self): + fv = _make_fv("fv") + fv.created_timestamp = None + obj = build_vector_store_object("proj", fv) + assert obj["created_at"] == 0 + + +class TestVectorStoreRegistry: + def _mock_store(self, feature_views: list, project: str = "test_project"): + store = unittest.mock.MagicMock() + store.project = project + store.list_feature_views.return_value = feature_views + return store + + def test_resolve_found(self): + fv = _make_fv("products") + store = self._mock_store([fv]) + registry = VectorStoreRegistry(store) + vs_id = feature_view_to_vs_id("test_project", "products") + assert registry.resolve(vs_id).name == "products" + + def test_resolve_not_found(self): + store = self._mock_store([_make_fv("products")]) + registry = VectorStoreRegistry(store) + with pytest.raises(FeatureViewNotFoundException): + registry.resolve("vs_does_not_exist_000000") + + def test_list_filters_non_vector_fvs(self): + vec_fv = _make_fv("vec_fv", vector_index=True) + plain_fv = _make_fv("plain_fv", vector_index=False) + store = self._mock_store([vec_fv, plain_fv]) + registry = VectorStoreRegistry(store) + stores = registry.list_vector_stores() + names = [s["name"] for s in stores] + assert "vec_fv" in names + assert "plain_fv" not in names + + def test_get_vector_store_found(self): + fv = _make_fv("embeddings") + store = self._mock_store([fv]) + registry = VectorStoreRegistry(store) + vs_id = feature_view_to_vs_id("test_project", "embeddings") + obj = registry.get_vector_store(vs_id) + assert obj is not None + assert obj["name"] == "embeddings" + + def test_get_vector_store_not_found(self): + store = self._mock_store([_make_fv("embeddings")]) + registry = VectorStoreRegistry(store) + assert registry.get_vector_store("vs_bogus_id_000000000000") is None + + def test_refresh_picks_up_new_fvs(self): + fv1 = _make_fv("fv1") + store = self._mock_store([fv1]) + registry = VectorStoreRegistry(store) + assert len(registry.list_vector_stores()) == 1 + + fv2 = _make_fv("fv2") + store.list_feature_views.return_value = [fv1, fv2] + registry.refresh() + assert len(registry.list_vector_stores()) == 2 diff --git a/sdk/python/tests/universal/feature_repos/universal/online_store/postgres.py b/sdk/python/tests/universal/feature_repos/universal/online_store/postgres.py index 0141232f12f..1a2289cef84 100644 --- a/sdk/python/tests/universal/feature_repos/universal/online_store/postgres.py +++ b/sdk/python/tests/universal/feature_repos/universal/online_store/postgres.py @@ -73,6 +73,7 @@ def create_online_store(self) -> Dict[str, Any]: "password": "test!@#$%", "database": "test", "vector_enabled": True, + "enable_openai_compatible_store": True, "port": self.container.get_exposed_port(5432), "sslmode": "disable", } diff --git a/ui/src/pages/saved-data-sets/CreateDatasetForm.tsx b/ui/src/pages/saved-data-sets/CreateDatasetForm.tsx index a3be0bae8df..7260f0593ed 100644 --- a/ui/src/pages/saved-data-sets/CreateDatasetForm.tsx +++ b/ui/src/pages/saved-data-sets/CreateDatasetForm.tsx @@ -103,6 +103,9 @@ const CreateDatasetForm = ({ onClose }: CreateDatasetFormProps) => { // Step 3: Storage & metadata const [datasetName, setDatasetName] = useState(""); + const [namespace, setNamespace] = useState(""); + const [collection, setCollection] = useState(""); + const [description, setDescription] = useState(""); const [storageType, setStorageType] = useState("file"); const [storagePath, setStoragePath] = useState(""); const [tags, setTags] = useState([]); @@ -268,6 +271,10 @@ const CreateDatasetForm = ({ onClose }: CreateDatasetFormProps) => { ), }; + if (namespace.trim()) payload.namespace = namespace.trim(); + if (collection.trim()) payload.collection = collection.trim(); + if (description.trim()) payload.description = description.trim(); + if (featureMode === "service" && selectedService.length > 0) { payload.feature_service_name = selectedService[0].label; } else if (featureMode === "individual" && selectedFeatures.length > 0) { @@ -562,6 +569,47 @@ const CreateDatasetForm = ({ onClose }: CreateDatasetFormProps) => { /> + + setDescription(e.target.value)} + placeholder="e.g. Training data for driver fraud model" + fullWidth + /> + + + + + + setNamespace(e.target.value)} + placeholder="e.g. fraud" + /> + + + + + setCollection(e.target.value)} + placeholder="e.g. training" + /> + + + + + + diff --git a/ui/src/pages/saved-data-sets/DatasetCatalogBrowser.tsx b/ui/src/pages/saved-data-sets/DatasetCatalogBrowser.tsx new file mode 100644 index 00000000000..bcb2bd844a5 --- /dev/null +++ b/ui/src/pages/saved-data-sets/DatasetCatalogBrowser.tsx @@ -0,0 +1,846 @@ +import React, { useCallback, useMemo, useState } from "react"; +import { + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiTitle, + EuiText, + EuiBadge, + EuiSpacer, + EuiIcon, + EuiBreadcrumbs, + EuiEmptyPrompt, + EuiTreeView, + EuiToolTip, + EuiButtonIcon, + EuiCopy, +} from "@elastic/eui"; +import type { Node as EuiTreeNode } from "@elastic/eui/src/components/tree_view/tree_view"; +import { useNavigate, useParams } from "react-router-dom"; + +/* ────────────────────────── types ────────────────────────── */ + +interface BrowsePath { + namespace?: string; + collection?: string; +} + +interface DatasetCatalogBrowserProps { + datasets: any[]; + onDelete?: (name: string) => void; +} + +/* ──────────────────── hierarchy builder ──────────────────── */ + +function buildHierarchy(datasets: any[]) { + const tree: Record> = {}; + for (const ds of datasets) { + const ns = ds.spec?.namespace || ""; + const col = ds.spec?.collection || ""; + if (!tree[ns]) tree[ns] = {}; + if (!tree[ns][col]) tree[ns][col] = []; + tree[ns][col].push(ds); + } + return tree; +} + +/* ──────────────────── colors ──────────────────── */ + +const NS_COLOR = "#0077CC"; +const COL_COLOR = "#8B5CF6"; + +const STORAGE_TYPE_CONFIG: Record< + string, + { label: string; color: string; icon: string } +> = { + file: { label: "File", color: "#4CAF50", icon: "document" }, + bigquery: { label: "BigQuery", color: "#4285F4", icon: "storage" }, + snowflake: { label: "Snowflake", color: "#29B5E8", icon: "snowflake" }, + redshift: { label: "Redshift", color: "#205B97", icon: "compute" }, + spark: { label: "Spark", color: "#E25A1C", icon: "bolt" }, + trino: { label: "Trino", color: "#DD00A1", icon: "database" }, + athena: { label: "Athena", color: "#8C4FFF", icon: "database" }, + custom: { label: "Custom", color: "#607D8B", icon: "gear" }, + unknown: { label: "Storage", color: "#98A2B3", icon: "database" }, +}; + +function detectStorageType(dataset: any): string { + const storage = dataset?.spec?.storage; + if (!storage) return "unknown"; + if (storage.fileStorage) return "file"; + if (storage.bigqueryStorage) return "bigquery"; + if (storage.snowflakeStorage) return "snowflake"; + if (storage.redshiftStorage) return "redshift"; + if (storage.sparkStorage) return "spark"; + if (storage.trinoStorage) return "trino"; + if (storage.athenaStorage) return "athena"; + if (storage.customStorage) return "custom"; + return "unknown"; +} + +function extractStoragePath(dataset: any): string | undefined { + const storage = dataset?.spec?.storage; + if (!storage) return undefined; + if (storage.fileStorage?.uri) return storage.fileStorage.uri; + if (storage.bigqueryStorage?.table) return storage.bigqueryStorage.table; + if (storage.snowflakeStorage?.table) return storage.snowflakeStorage.table; + if (storage.redshiftStorage?.table) return storage.redshiftStorage.table; + if (storage.sparkStorage?.path) return storage.sparkStorage.path; + if (storage.sparkStorage?.table) return storage.sparkStorage.table; + if (storage.trinoStorage?.table) return storage.trinoStorage.table; + if (storage.athenaStorage?.table) return storage.athenaStorage.table; + if (storage.customStorage?.configuration) + return storage.customStorage.configuration; + return undefined; +} + +function truncatePath(path: string, maxLen: number = 42): string { + if (path.length <= maxLen) return path; + return `${path.slice(0, 18)}...${path.slice(-20)}`; +} + +function getRelativeTime(date: Date): string { + const diffMs = Date.now() - date.getTime(); + const diffDays = Math.floor(diffMs / 86400000); + if (diffDays === 0) return "Today"; + if (diffDays === 1) return "Yesterday"; + if (diffDays < 7) return `${diffDays}d ago`; + if (diffDays < 30) return `${Math.floor(diffDays / 7)}w ago`; + if (diffDays < 365) return `${Math.floor(diffDays / 30)}mo ago`; + return `${Math.floor(diffDays / 365)}y ago`; +} + +/* ──────────────────── tile: dataset card ──────────────────── */ + +const DatasetCard: React.FC<{ + dataset: any; + onNavigate: () => void; + onDelete?: (name: string) => void; +}> = ({ dataset, onNavigate, onDelete }) => { + const [isHovered, setIsHovered] = useState(false); + const spec = dataset.spec || {}; + const meta = dataset.meta || {}; + const name = spec.name || "unknown"; + const features = spec.features || []; + const joinKeys = spec.joinKeys || spec.join_keys || []; + const tags = spec.tags || {}; + const description = spec.description || ""; + const featureServiceName = + spec.featureServiceName || spec.feature_service_name; + const createdTimestamp = meta.createdTimestamp || meta.created_timestamp; + const storagePath = extractStoragePath(dataset); + const storageType = detectStorageType(dataset); + const storageInfo = + STORAGE_TYPE_CONFIG[storageType] || STORAGE_TYPE_CONFIG.unknown; + + const formattedDate = createdTimestamp + ? new Date(createdTimestamp).toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + }) + : null; + const relativeTime = createdTimestamp + ? getRelativeTime(new Date(createdTimestamp)) + : null; + + const codeSnippet = `dataset = store.get_saved_dataset("${name}")\ndf = dataset.to_df()`; + + return ( + + setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + onClick={onNavigate} + style={{ + height: "100%", + display: "flex", + flexDirection: "column", + transition: "all 0.2s ease", + transform: isHovered ? "translateY(-2px)" : "none", + borderTop: `3px solid ${storageInfo.color}`, + cursor: "pointer", + }} + > + {/* Header */} + + + +

+ {name} +

+ + + + + {storageInfo.label} + + + + + {/* Description */} + {description && ( + +

{description}

+ + )} + + + + {/* Storage path */} + {storagePath && ( + + + {" "} + + {truncatePath(storagePath)} + + + + )} + + + + {/* Metrics */} + + + + Features + + + {features.length} + + + + + Retrieval Keys + + + {joinKeys.length} + + + {featureServiceName && ( + + + Service + + + {featureServiceName} + + + )} + + +
+ + + {/* Tags */} + {Object.keys(tags).length > 0 && ( + <> + + {Object.entries(tags) + .slice(0, 3) + .map(([key, value]) => ( + + + {key}: {value as string} + + + ))} + {Object.keys(tags).length > 3 && ( + + + +{Object.keys(tags).length - 3} more + + + )} + + + + )} + + {/* Footer */} + + + {formattedDate && ( + + + {relativeTime} + + + )} + + + + + + {(copy) => ( + + { + e.stopPropagation(); + copy(); + }} + /> + + )} + + + {onDelete && ( + + + { + e.stopPropagation(); + onDelete(name); + }} + /> + + + )} + + + + + + ); +}; + +/* ──────────────────── tile: folder card (same width as dataset) ──────────────────── */ + +const FolderCard: React.FC<{ + name: string; + type: "namespace" | "collection"; + datasetCount: number; + collectionCount?: number; + onClick: () => void; +}> = ({ name, type, datasetCount, collectionCount, onClick }) => { + const [isHovered, setIsHovered] = useState(false); + const color = type === "namespace" ? NS_COLOR : COL_COLOR; + + return ( + + setIsHovered(true)} + onMouseLeave={() => setIsHovered(false)} + onClick={onClick} + style={{ + cursor: "pointer", + transition: "all 0.2s ease", + transform: isHovered ? "translateY(-2px)" : "none", + borderTop: `3px solid ${color}`, + display: "flex", + flexDirection: "column", + }} + > + + + + + + +

{name}

+ + + + + + + + {type === "namespace" && + collectionCount !== undefined && + collectionCount > 0 && ( + + + Collections + + + {collectionCount} + + + )} + + + Datasets + + + {datasetCount} + + + + + + ); +}; + +/* ──────────────────── tree styles ──────────────────── */ + +const TREE_CSS = ` + .catalogTree .euiTreeView__node { + margin-bottom: 1px; + } + .catalogTree .euiTreeView__node--expanded > .euiTreeView__nodeInner { + background: rgba(0, 119, 204, 0.05); + border-radius: 4px; + } + .catalogTree .euiTreeView__nodeInner { + padding: 4px 6px; + border-radius: 4px; + transition: background 0.15s ease; + } + .catalogTree .euiTreeView__nodeInner:hover { + background: rgba(0, 119, 204, 0.08); + } + .catalogTree .euiTreeView__nodeInner--withArrow .euiTreeView__nodeInner__arrow { + margin-right: 2px; + } + .catalogTree .euiTreeView__nodeLabel { + font-size: 13px; + } +`; + +/* ──────────────────── main component ──────────────────── */ + +const DatasetCatalogBrowser: React.FC = ({ + datasets, + onDelete, +}) => { + const { projectName } = useParams(); + const navigate = useNavigate(); + const [browsePath, setBrowsePath] = useState({}); + const [showTree, setShowTree] = useState(true); + + const hierarchy = useMemo(() => buildHierarchy(datasets), [datasets]); + + const namespaceList = useMemo(() => { + const entries: Array<{ + name: string; + collections: string[]; + totalDatasets: number; + }> = []; + for (const [ns, cols] of Object.entries(hierarchy)) { + if (ns === "") continue; + const colNames = Object.keys(cols) + .filter((c) => c !== "") + .sort(); + const total = Object.values(cols).reduce((s, a) => s + a.length, 0); + entries.push({ name: ns, collections: colNames, totalDatasets: total }); + } + entries.sort((a, b) => a.name.localeCompare(b.name)); + return entries; + }, [hierarchy]); + + const rootDatasets = useMemo(() => { + const cols = hierarchy[""]; + if (!cols) return []; + const all: any[] = []; + for (const arr of Object.values(cols)) all.push(...arr); + return all; + }, [hierarchy]); + + const goToDataset = useCallback( + (dataset: any) => { + const p = dataset.project || dataset.spec?.project || projectName; + const n = dataset.spec?.name || dataset.name; + navigate(`/p/${p}/data-set/${n}`); + }, + [navigate, projectName], + ); + + /* ── tree sidebar (EuiTreeView) ── */ + + // Clickable label for parent nodes — onClick navigates, stopPropagation prevents toggle + const navLabel = useCallback( + ( + text: string, + onClick: () => void, + isActive: boolean, + activeColor?: string, + ) => ( + { + e.stopPropagation(); + onClick(); + }} + style={{ + cursor: "pointer", + fontWeight: isActive ? 600 : 400, + color: isActive ? activeColor || NS_COLOR : undefined, + }} + > + {text} + + ), + [], + ); + + const treeItems: EuiTreeNode[] = useMemo(() => { + const isRootSelected = browsePath.namespace === undefined; + const items: EuiTreeNode[] = [ + { + id: "_root", + label: navLabel( + "All Datasets", + () => setBrowsePath({}), + isRootSelected, + ), + icon: , + isExpanded: true, + }, + ]; + + for (const ns of namespaceList) { + const nsData = hierarchy[ns.name] || {}; + const nsChildren: EuiTreeNode[] = []; + const nsSelected = + browsePath.namespace === ns.name && !browsePath.collection; + + // Collection sub-folders + for (const col of ns.collections) { + const colDatasets = nsData[col] || []; + const colSelected = + browsePath.namespace === ns.name && browsePath.collection === col; + + const dsLeaves: EuiTreeNode[] = colDatasets.map((ds: any) => ({ + id: `ds:${ns.name}/${col}/${ds.spec?.name || ds.name}`, + label: ds.spec?.name || ds.name || "unknown", + icon: , + callback: () => { + goToDataset(ds); + return `ds:${ns.name}/${col}/${ds.spec?.name}`; + }, + })); + + nsChildren.push({ + id: `col:${ns.name}/${col}`, + label: navLabel( + `${col} (${colDatasets.length})`, + () => setBrowsePath({ namespace: ns.name, collection: col }), + colSelected, + COL_COLOR, + ), + icon: , + iconWhenExpanded: ( + + ), + children: dsLeaves.length > 0 ? dsLeaves : undefined, + }); + } + + // Direct datasets under namespace (no collection) + const directDatasets = nsData[""] || []; + for (const ds of directDatasets) { + nsChildren.push({ + id: `ds:${ns.name}/_/${ds.spec?.name || ds.name}`, + label: ds.spec?.name || ds.name || "unknown", + icon: , + callback: () => { + goToDataset(ds); + return `ds:${ns.name}/_/${ds.spec?.name}`; + }, + }); + } + + items.push({ + id: `ns:${ns.name}`, + label: navLabel( + `${ns.name} (${ns.totalDatasets})`, + () => setBrowsePath({ namespace: ns.name }), + nsSelected, + NS_COLOR, + ), + icon: , + iconWhenExpanded: ( + + ), + children: nsChildren.length > 0 ? nsChildren : undefined, + }); + } + + // Ungrouped datasets + if (rootDatasets.length > 0) { + const ungroupedLeaves: EuiTreeNode[] = rootDatasets.map((ds: any) => ({ + id: `ds:_ungrouped/${ds.spec?.name || ds.name}`, + label: ds.spec?.name || ds.name || "unknown", + icon: , + callback: () => { + goToDataset(ds); + return `ds:_ungrouped/${ds.spec?.name}`; + }, + })); + + items.push({ + id: "_ungrouped", + label: navLabel( + `Ungrouped (${rootDatasets.length})`, + () => setBrowsePath({}), + false, + "#98A2B3", + ), + icon: , + children: ungroupedLeaves, + }); + } + + return items; + }, [ + namespaceList, + rootDatasets, + hierarchy, + browsePath, + goToDataset, + navLabel, + ]); + + /* ── breadcrumbs ── */ + const breadcrumbs = useMemo(() => { + const crumbs: Array<{ text: string; onClick?: () => void }> = [ + { + text: "All Datasets", + onClick: + browsePath.namespace !== undefined + ? () => setBrowsePath({}) + : undefined, + }, + ]; + if (browsePath.namespace) { + crumbs.push({ + text: browsePath.namespace, + onClick: browsePath.collection + ? () => setBrowsePath({ namespace: browsePath.namespace }) + : undefined, + }); + if (browsePath.collection) { + crumbs.push({ text: browsePath.collection }); + } + } + return crumbs; + }, [browsePath]); + + /* ── content ── */ + const renderContent = () => { + // ─── Root level: namespace folders + datasets mixed in one grid ─── + if (browsePath.namespace === undefined) { + if (namespaceList.length === 0 && rootDatasets.length === 0) { + return ( + No datasets yet} + body={ +

+ Add datasets and organize them into namespaces and collections. +

+ } + /> + ); + } + + return ( + + {namespaceList.map((ns) => ( + setBrowsePath({ namespace: ns.name })} + /> + ))} + {rootDatasets.map((d: any) => ( + goToDataset(d)} + onDelete={onDelete} + /> + ))} + + ); + } + + // ─── Namespace level: collection folders + direct datasets mixed ─── + if (browsePath.namespace && !browsePath.collection) { + const ns = browsePath.namespace; + const cols = hierarchy[ns] || {}; + const colNames = Object.keys(cols) + .filter((c) => c !== "") + .sort(); + const directDs = cols[""] || []; + + return ( + + {colNames.map((col) => ( + setBrowsePath({ namespace: ns, collection: col })} + /> + ))} + {directDs.map((d: any) => ( + goToDataset(d)} + onDelete={onDelete} + /> + ))} + {colNames.length === 0 && directDs.length === 0 && ( + + + This namespace is empty. + + + )} + + ); + } + + // ─── Collection level: datasets only ─── + if (browsePath.namespace && browsePath.collection) { + const dsList = + hierarchy[browsePath.namespace]?.[browsePath.collection] || []; + + return ( + + {dsList.map((d: any) => ( + goToDataset(d)} + onDelete={onDelete} + /> + ))} + {dsList.length === 0 && ( + + + No datasets in this collection. + + + )} + + ); + } + + return null; + }; + + /* ──────────────────── render ──────────────────── */ + + const hasTree = namespaceList.length > 0; + + return ( +
+ {/* Tree sidebar */} + {hasTree && ( +
+ {showTree ? ( + <> +
+ + + Catalog + + + + setShowTree(false)} + /> + +
+ +
+ + +
+ + ) : ( + + setShowTree(true)} + /> + + )} +
+ )} + + {/* Content */} +
+ + + {renderContent()} +
+
+ ); +}; + +export default DatasetCatalogBrowser; diff --git a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx index b765da70241..b0d8fc61f28 100644 --- a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx +++ b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx @@ -96,6 +96,9 @@ const DatasetOverviewTab = () => { const tags = data.spec?.tags || {}; const featureServiceName = data.spec?.featureServiceName || data.spec?.feature_service_name; + const namespace = data.spec?.namespace || ""; + const collection = data.spec?.collection || ""; + const description = data.spec?.description || ""; const createdTs = data.meta?.createdTimestamp || data.meta?.created_timestamp; const minEventTs = data.meta?.minEventTimestamp || data.meta?.min_event_timestamp; @@ -168,6 +171,33 @@ const DatasetOverviewTab = () => { + {description && ( + <> + Description + + {description} + + + )} + + {namespace && ( + <> + Namespace + + {namespace} + + + )} + + {collection && ( + <> + Collection + + {collection} + + + )} + Storage Type {storageInfo.type} diff --git a/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx b/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx index 2c96a0ee109..5b6c33038ff 100644 --- a/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx +++ b/ui/src/pages/saved-data-sets/DatasetsCardGrid.tsx @@ -103,6 +103,9 @@ const DatasetCard: React.FC = ({ const tags = spec.tags || {}; const featureServiceName = spec.featureServiceName || spec.feature_service_name; + const namespace = spec.namespace || ""; + const collection = spec.collection || ""; + const description = spec.description || ""; const createdTimestamp = meta.createdTimestamp || meta.created_timestamp; const storagePath = extractStoragePath(dataset); const storageType = detectStorageType(dataset); @@ -169,6 +172,32 @@ const DatasetCard: React.FC = ({ + {/* Description */} + {description && ( + +

{description}

+ + )} + + {/* Namespace / Collection badges */} + {(namespace || collection) && ( + <> + + + {namespace && ( + + {namespace} + + )} + {collection && ( + + {collection} + + )} + + + )} + {/* Storage path */} diff --git a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx index 823f42176f3..251af853ac7 100644 --- a/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx +++ b/ui/src/pages/saved-data-sets/DatasetsListingTable.tsx @@ -35,6 +35,44 @@ const DatasetsListingTable = ({ datasets }: DatasetsListingTableProps) => { ); }, }, + { + name: "Namespace", + render: (item: any) => { + const ns = item.spec?.namespace; + return ns ? ( + {ns} + ) : ( + — + ); + }, + width: "140px", + }, + { + name: "Collection", + render: (item: any) => { + const col = item.spec?.collection; + return col ? ( + {col} + ) : ( + — + ); + }, + width: "140px", + }, + { + name: "Description", + render: (item: any) => { + const desc = item.spec?.description; + return desc ? ( + + {desc.length > 50 ? desc.slice(0, 50) + "…" : desc} + + ) : ( + — + ); + }, + width: "200px", + }, { name: "Features", render: (item: any) => (item.spec?.features || []).length, diff --git a/ui/src/pages/saved-data-sets/EditDatasetModal.tsx b/ui/src/pages/saved-data-sets/EditDatasetModal.tsx index db567a15703..77c938e3973 100644 --- a/ui/src/pages/saved-data-sets/EditDatasetModal.tsx +++ b/ui/src/pages/saved-data-sets/EditDatasetModal.tsx @@ -307,6 +307,9 @@ const EditDatasetModal = ({ // Form state const [storagePath, setStoragePath] = useState(extractStoragePath(dataset)); const [storageType, setStorageType] = useState(detectStorageType(dataset)); + const [namespace, setNamespace] = useState(spec.namespace || ""); + const [collection, setCollection] = useState(spec.collection || ""); + const [description, setDescription] = useState(spec.description || ""); const [featuresInput, setFeaturesInput] = useState( (spec.features || []).map((f: string) => ({ label: f })), ); @@ -368,6 +371,9 @@ const EditDatasetModal = ({ tags: tagsObj, full_feature_names: fullFeatureNames, feature_service_name: featureServiceName || undefined, + namespace: namespace.trim() || undefined, + collection: collection.trim() || undefined, + description: description.trim() || undefined, allow_override: true, }; await onSubmit(payload); @@ -421,6 +427,20 @@ const EditDatasetModal = ({ + + + setDescription(e.target.value)} + placeholder="e.g. Training data for driver fraud model" + /> + + + + + + + {/* Organization */} + +

Organization (optional)

+ + + + + + + setNamespace(e.target.value)} + placeholder="e.g. fraud" + /> + + + + + setCollection(e.target.value)} + placeholder="e.g. training" + /> + + + + diff --git a/ui/src/pages/saved-data-sets/Index.tsx b/ui/src/pages/saved-data-sets/Index.tsx index 78770c5aaf8..1c587d83883 100644 --- a/ui/src/pages/saved-data-sets/Index.tsx +++ b/ui/src/pages/saved-data-sets/Index.tsx @@ -16,7 +16,6 @@ import { EuiFlexGroup, EuiFlexItem, EuiFieldSearch, - EuiStat, EuiPanel, EuiSelect, EuiText, @@ -32,6 +31,7 @@ import { DatasetIcon } from "../../graphics/DatasetIcon"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; import DatasetsCardGrid from "./DatasetsCardGrid"; import DatasetsListingTable from "./DatasetsListingTable"; +import DatasetCatalogBrowser from "./DatasetCatalogBrowser"; import DatasetsIndexEmptyState from "./DatasetsIndexEmptyState"; import AddToCatalogModal from "./AddToCatalogModal"; import type { RegisterDatasetPayload } from "./RegisterDatasetModal"; @@ -62,6 +62,7 @@ const SORT_OPTIONS = [ ]; const VIEW_TOGGLE_BUTTONS = [ + { id: "catalog", label: "Catalog", iconType: "folderClosed" }, { id: "cards", label: "Cards", iconType: "grid" }, { id: "table", label: "Table", iconType: "list" }, ]; @@ -84,7 +85,8 @@ const Index = () => { const [deleteTarget, setDeleteTarget] = useState(null); const [searchQuery, setSearchQuery] = useState(""); const [sortBy, setSortBy] = useState("created_desc"); - const [viewMode, setViewMode] = useState("cards"); + const [viewMode, setViewMode] = useState("catalog"); + const [namespaceFilter, setNamespaceFilter] = useState("all"); const [submitError, setSubmitError] = useState(null); const [successMessage, setSuccessMessage] = useState(null); @@ -197,12 +199,18 @@ const Index = () => { // Compute summary stats const stats = useMemo(() => { if (!data) - return { total: 0, totalFeatures: 0, storageTypes: new Set() }; + return { + total: 0, + totalFeatures: 0, + storageTypes: new Set(), + namespaces: [] as string[], + }; const totalFeatures = data.reduce( (acc: number, ds: any) => acc + (ds.spec?.features?.length || 0), 0, ); const storageTypes = new Set(); + const namespacesSet = new Set(); data.forEach((ds: any) => { const storage = ds.spec?.storage; if (storage?.fileStorage) storageTypes.add("File"); @@ -213,8 +221,11 @@ const Index = () => { else if (storage?.trinoStorage) storageTypes.add("Trino"); else if (storage?.athenaStorage) storageTypes.add("Athena"); else if (storage?.customStorage) storageTypes.add("Custom"); + const ns = ds.spec?.namespace; + if (ns) namespacesSet.add(ns); }); - return { total: data.length, totalFeatures, storageTypes }; + const namespaces = Array.from(namespacesSet).sort(); + return { total: data.length, totalFeatures, storageTypes, namespaces }; }, [data]); // Filter and sort @@ -222,9 +233,20 @@ const Index = () => { if (!data) return []; let filtered = data; + // Namespace filter + if (namespaceFilter !== "all") { + if (namespaceFilter === "_none") { + filtered = filtered.filter((ds: any) => !ds.spec?.namespace); + } else { + filtered = filtered.filter( + (ds: any) => ds.spec?.namespace === namespaceFilter, + ); + } + } + if (searchQuery.trim()) { const q = searchQuery.toLowerCase(); - filtered = data.filter((ds: any) => { + filtered = filtered.filter((ds: any) => { const name = (ds.spec?.name || "").toLowerCase(); const tags = JSON.stringify(ds.spec?.tags || {}).toLowerCase(); const features = (ds.spec?.features || []).join(" ").toLowerCase(); @@ -233,11 +255,17 @@ const Index = () => { ds.spec?.feature_service_name || "" ).toLowerCase(); + const ns = (ds.spec?.namespace || "").toLowerCase(); + const col = (ds.spec?.collection || "").toLowerCase(); + const desc = (ds.spec?.description || "").toLowerCase(); return ( name.includes(q) || tags.includes(q) || features.includes(q) || - service.includes(q) + service.includes(q) || + ns.includes(q) || + col.includes(q) || + desc.includes(q) ); }); } @@ -254,7 +282,7 @@ const Index = () => { }); return filtered; - }, [data, searchQuery, sortBy]); + }, [data, searchQuery, sortBy, namespaceFilter]); const hasData = data && data.length > 0; @@ -434,53 +462,71 @@ const Index = () => { {isSuccess && hasData && ( <> - {/* Summary Stats */} - - - - - - - - - - + {/* View mode toggle — always visible */} + + + + + + {stats.total} datasets + {stats.namespaces.length > 0 && ( + <> + {" "} + across {stats.namespaces.length}{" "} + namespaces + + )} + + + - - - - + + setViewMode(id)} + isIconOnly + buttonSize="compressed" + /> - + - {/* Search + Sort + View Toggle */} + {/* Search + Namespace Filter + Sort — shared toolbar */} setSearchQuery(e.target.value)} isClearable fullWidth /> + {stats.namespaces.length > 0 && ( + + ({ + value: ns, + text: ns, + })), + ]} + value={namespaceFilter} + onChange={(e) => setNamespaceFilter(e.target.value)} + compressed + prepend="Namespace" + /> + + )} { prepend="Sort" /> - - setViewMode(id)} - isIconOnly - buttonSize="compressed" - /> - - + - {/* Results count when searching */} - {searchQuery.trim() && ( + {/* Results count when filtering */} + {(searchQuery.trim() || namespaceFilter !== "all") && ( <> Showing {processedData.length} of {data.length} datasets + {namespaceFilter !== "all" && namespaceFilter !== "_none" && ( + <> + {" "} + in namespace {namespaceFilter} + + )} + {namespaceFilter === "_none" && <> with no namespace} )} - {/* Content */} - {viewMode === "cards" ? ( + {/* Catalog (hierarchical) view */} + {viewMode === "catalog" && ( + setDeleteTarget(name)} + /> + )} + + {/* Flat views (cards / table) */} + {viewMode === "cards" && ( setDeleteTarget(name)} /> - ) : ( + )} + {viewMode === "table" && ( )} diff --git a/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx b/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx index 90f93b8ca31..fa4013d661c 100644 --- a/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx +++ b/ui/src/pages/saved-data-sets/RegisterDatasetModal.tsx @@ -37,6 +37,9 @@ export interface RegisterDatasetPayload { tags: Record; full_feature_names: boolean; feature_service_name?: string; + namespace?: string; + collection?: string; + description?: string; } interface RegisterDatasetModalProps { @@ -241,6 +244,9 @@ const RegisterDatasetModal = ({ // Form state const [name, setName] = useState(""); + const [namespace, setNamespace] = useState(""); + const [collection, setCollection] = useState(""); + const [description, setDescription] = useState(""); const [storagePath, setStoragePath] = useState(""); const [storageType, setStorageType] = useState("file"); const [featuresInput, setFeaturesInput] = useState( @@ -339,6 +345,9 @@ const RegisterDatasetModal = ({ tags: tagsObj, full_feature_names: fullFeatureNames, feature_service_name: featureServiceName || undefined, + namespace: namespace.trim() || undefined, + collection: collection.trim() || undefined, + description: description.trim() || undefined, }; await onSubmit(payload); }; @@ -395,6 +404,18 @@ const RegisterDatasetModal = ({ /> + + + setDescription(e.target.value)} + placeholder="e.g. Training data for driver fraud model" + /> + + + + + {/* Section: Organization */} + +

Organization (optional)

+ + + + + + Group datasets into namespaces and collections for hierarchical + organization. Leave empty to keep the dataset at the top level. + + + + + + + + setNamespace(e.target.value)} + placeholder="e.g. fraud" + /> + + + + + setCollection(e.target.value)} + placeholder="e.g. training" + /> + + + +