Skip to content

Commit 3656727

Browse files
HaoXuAIHaoXuAI
authored
Merge branch 'master' into on-demand-feature-view-aggregation
2 parents be2a2a3 + 56c5910 commit 3656727

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+9314
-7110
lines changed

‎.github/workflows/registry-rest-api-tests.yml‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
# .github/workflows/registry-rest-api-tests.yml
2+
name: pr-rest-API-tests
3+
4+
on:
5+
push:
6+
branches:
7+
- main
8+
pull_request:
9+
types:
10+
- opened
11+
- synchronize
12+
- labeled
13+
14+
jobs:
15+
registry-rest-api-tests:
16+
timeout-minutes: 30
17+
if:
18+
((github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'lgtm' || github.event.label.name == 'ok-to-test')) ||
19+
(github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))) &&
20+
github.repository == 'feast-dev/feast'
21+
runs-on: ubuntu-latest
22+
23+
services:
24+
kind:
25+
# Specify the Kubernetes version
26+
image: kindest/node:v1.30.6
27+
28+
env:
29+
KIND_CLUSTER: "registry-rest-api-cluster"
30+
31+
steps:
32+
- name: Checkout code
33+
uses: actions/checkout@v4
34+
35+
- name: Free Disk Space (Ubuntu)
36+
uses: jlumbroso/free-disk-space@v1.3.1
37+
with:
38+
android: true
39+
dotnet: true
40+
haskell: true
41+
large-packages: false
42+
docker-images: false
43+
swap-storage: false
44+
tool-cache: false
45+
46+
- name: Set up Go
47+
uses: actions/setup-go@v5
48+
with:
49+
go-version: 1.22.9
50+
51+
- name: Create KIND cluster
52+
run: |
53+
cat <<EOF | kind create cluster --name $KIND_CLUSTER --wait 10m --config=-
54+
kind: Cluster
55+
apiVersion: kind.x-k8s.io/v1alpha4
56+
nodes:
57+
- role: control-plane
58+
extraMounts:
59+
- hostPath: /mnt/kind
60+
containerPath: /var/lib/containerd
61+
extraPortMappings:
62+
- containerPort: 80
63+
hostPort: 80
64+
protocol: TCP
65+
- containerPort: 443
66+
hostPort: 443
67+
protocol: TCP
68+
kubeadmConfigPatches:
69+
- |
70+
kind: InitConfiguration
71+
nodeRegistration:
72+
kubeletExtraArgs:
73+
node-labels: "ingress-ready=true"
74+
EOF
75+
76+
- name: Set up kubernetes context
77+
run: |
78+
kubectl config use-context kind-$KIND_CLUSTER
79+
echo "kind context is switched to cluster kind-$KIND_CLUSTER"
80+
81+
82+
- name: Set up Ingress controller
83+
run: |
84+
echo "Installing ingress-nginx for KIND..."
85+
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.9.5/deploy/static/provider/kind/deploy.yaml
86+
87+
echo "⏳ Waiting for ingress controller to become ready..."
88+
kubectl wait --namespace ingress-nginx \
89+
--for=condition=Ready pod \
90+
--selector=app.kubernetes.io/component=controller \
91+
--timeout=180s
92+
93+
- name: Add ingress DNS to /etc/hosts
94+
run: |
95+
echo "127.0.0.1 feast.kind.test" | sudo tee -a /etc/hosts
96+
echo "Added 'feast.kind.test' to /etc/hosts"
97+
98+
- name: Build and Deploy Feast Operator images
99+
run: |
100+
# Create namespace
101+
kubectl create ns feast-operator-system || true
102+
103+
# navigate to feast operator path
104+
cd infra/feast-operator/
105+
106+
# Build Feast Operator Docker image
107+
make docker-build IMG=localhost/feast-operator:v0.0.1
108+
109+
# Load Operator image into KIND
110+
kind load docker-image localhost/feast-operator:v0.0.1 --name $KIND_CLUSTER
111+
112+
# Build Feast dev image
113+
make feast-ci-dev-docker-img
114+
115+
# Tag Feast image for KIND compatibility
116+
docker tag feastdev/feature-server:dev localhost/feastdev/feature-server:dev
117+
118+
# Load Feast image into KIND
119+
kind load docker-image localhost/feastdev/feature-server:dev --name $KIND_CLUSTER
120+
121+
# Install CRDs
122+
make install
123+
124+
# Deploy operator to the KIND cluster
125+
make deploy IMG=localhost/feast-operator:v0.0.1 FS_IMG=localhost/feastdev/feature-server:dev
126+
127+
# Wait for controller manager to be ready
128+
kubectl wait deployment feast-operator-controller-manager -n feast-operator-system --for=condition=Available=True --timeout=180s
129+
130+
- name: Setup Python
131+
uses: actions/setup-python@v5
132+
id: setup-python
133+
with:
134+
python-version: 3.11
135+
architecture: x64
136+
137+
- name: Install the latest version of uv
138+
uses: astral-sh/setup-uv@v5
139+
with:
140+
enable-cache: true
141+
142+
- name: Install dependencies
143+
run: make install-python-dependencies-ci
144+
145+
- name: Setup and Run Registry Rest API tests
146+
run: |
147+
echo "Running Registry REST API tests..."
148+
cd sdk/python/tests/registry_rest_api_tests/
149+
pytest test_feast_registry.py -s
150+
151+
- name: Clean up docker images
152+
if: always()
153+
run: |
154+
docker images --format '{{.Repository}}:{{.Tag}}' | grep 'feast' | xargs -r docker rmi -f
155+
docker system prune -a -f
156+
157+
- name: Debug KIND Cluster when there is a failure
158+
if: failure()
159+
run: |
160+
kubectl get pods --all-namespaces
161+
kubectl describe nodes
162+
163+
- name: Clean up
164+
if: always()
165+
run: |
166+
# Delete the KIND cluster after tests
167+
kind delete cluster --name kind-$KIND_CLUSTER
168+

‎pyproject.toml‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,12 @@ ibis = [
9090
ikv = [
9191
"ikvpy>=0.0.36",
9292
]
93+
image = [
94+
"feast[pytorch]",
95+
"timm>=0.6.0",
96+
"Pillow>=8.0.0",
97+
"scikit-learn>=1.0.0",
98+
]
9399
k8s = ["kubernetes<=20.13.0"]
94100
milvus = [
95101
"pymilvus==2.4.9",
@@ -168,9 +174,9 @@ ci = [
168174
"types-setuptools",
169175
"types-tabulate",
170176
"virtualenv<20.24.2",
171-
"feast[aws, azure, cassandra, clickhouse, couchbase, delta, docling, duckdb, elasticsearch, faiss, gcp, ge, go, grpcio, hazelcast, hbase, ibis, ikv, k8s, mcp, milvus, mssql, mysql, opentelemetry, spark, trino, postgres, pytorch, qdrant, rag, ray, redis, singlestore, snowflake, sqlite_vec]"
177+
"feast[aws, azure, cassandra, clickhouse, couchbase, delta, docling, duckdb, elasticsearch, faiss, gcp, ge, go, grpcio, hazelcast, hbase, ibis, ikv, image, k8s, mcp, milvus, mssql, mysql, opentelemetry, spark, trino, postgres, pytorch, qdrant, rag, ray, redis, singlestore, snowflake, sqlite_vec]"
172178
]
173-
nlp = ["feast[docling, milvus, pytorch, rag]"]
179+
nlp = ["feast[docling, image, milvus, pytorch, rag]"]
174180
dev = ["feast[ci]"]
175181
docs = ["feast[ci]"]
176182
# used for the 'feature-server' container image build

‎sdk/python/feast/feature_store.py‎

Lines changed: 103 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2239,6 +2239,12 @@ def retrieve_online_documents_v2(
22392239
query: Optional[List[float]] = None,
22402240
query_string: Optional[str] = None,
22412241
distance_metric: Optional[str] = "L2",
2242+
query_image_bytes: Optional[bytes] = None,
2243+
query_image_model: Optional[str] = "resnet34",
2244+
combine_with_text: bool = False,
2245+
text_weight: float = 0.5,
2246+
image_weight: float = 0.5,
2247+
combine_strategy: str = "weighted_sum",
22422248
) -> OnlineResponse:
22432249
"""
22442250
Retrieves the top k closest document features. Note, embeddings are a subset of features.
@@ -2247,13 +2253,105 @@ def retrieve_online_documents_v2(
22472253
features: The list of features that should be retrieved from the online document store. These features can be
22482254
specified either as a list of string document feature references or as a feature service. String feature
22492255
references must have format "feature_view:feature", e.g, "document_fv:document_embeddings".
2250-
query: The embeded query to retrieve the closest document features for (optional)
22512256
top_k: The number of closest document features to retrieve.
2257+
query_string: Text query for hybrid search (alternative to query parameter)
22522258
distance_metric: The distance metric to use for retrieval.
2253-
query_string: The query string to retrieve the closest document features using keyword search (bm25).
2259+
query_image_bytes: Query image as bytes (for image similarity search)
2260+
query_image_model: Model name for image embedding generation
2261+
combine_with_text: Whether to combine text and image embeddings for multi-modal search
2262+
text_weight: Weight for text embedding in combined search (0.0 to 1.0)
2263+
image_weight: Weight for image embedding in combined search (0.0 to 1.0)
2264+
combine_strategy: Strategy for combining embeddings ("weighted_sum", "concatenate", "average")
2265+
2266+
Returns:
2267+
OnlineResponse with similar documents and metadata
2268+
2269+
Examples:
2270+
Text search only::
2271+
2272+
results = store.retrieve_online_documents_v2(
2273+
features=["documents:embedding", "documents:title"],
2274+
query=[0.1, 0.2, 0.3], # text embedding vector
2275+
top_k=5
2276+
)
2277+
2278+
Image search only::
2279+
2280+
results = store.retrieve_online_documents_v2(
2281+
features=["images:embedding", "images:filename"],
2282+
query_image_bytes=b"image_data", # image bytes
2283+
top_k=5
2284+
)
2285+
2286+
Combined text + image search::
2287+
2288+
results = store.retrieve_online_documents_v2(
2289+
features=["documents:embedding", "documents:title"],
2290+
query=[0.1, 0.2, 0.3], # text embedding vector
2291+
query_image_bytes=b"image_data", # image bytes
2292+
combine_with_text=True,
2293+
text_weight=0.3,
2294+
image_weight=0.7,
2295+
top_k=5
2296+
)
22542297
"""
2255-
assert query is not None or query_string is not None, (
2256-
"Either query or query_string must be provided."
2298+
if query is None and not query_image_bytes and not query_string:
2299+
raise ValueError(
2300+
"Must provide either query (text embedding), "
2301+
"query_image_bytes, or query_string"
2302+
)
2303+
2304+
if combine_with_text and not (query is not None and query_image_bytes):
2305+
raise ValueError(
2306+
"combine_with_text=True requires both query (text embedding) "
2307+
"and query_image_bytes"
2308+
)
2309+
2310+
if combine_with_text and abs(text_weight + image_weight - 1.0) > 1e-6:
2311+
raise ValueError("text_weight + image_weight must equal 1.0 when combining")
2312+
2313+
image_embedding = None
2314+
if query_image_bytes is not None:
2315+
try:
2316+
from feast.image_utils import ImageFeatureExtractor
2317+
2318+
model_name = query_image_model or "resnet34"
2319+
extractor = ImageFeatureExtractor(model_name)
2320+
image_embedding = extractor.extract_embedding(query_image_bytes)
2321+
except ImportError:
2322+
raise ImportError(
2323+
"Image processing dependencies are not installed. "
2324+
"Please install with: pip install feast[image]"
2325+
)
2326+
2327+
text_embedding = query
2328+
2329+
if (
2330+
combine_with_text
2331+
and text_embedding is not None
2332+
and image_embedding is not None
2333+
):
2334+
# Combine text and image embeddings
2335+
from feast.image_utils import combine_embeddings
2336+
2337+
final_query = combine_embeddings(
2338+
text_embedding=text_embedding,
2339+
image_embedding=image_embedding,
2340+
strategy=combine_strategy,
2341+
text_weight=text_weight,
2342+
image_weight=image_weight,
2343+
)
2344+
elif image_embedding is not None:
2345+
final_query = image_embedding
2346+
elif text_embedding is not None:
2347+
final_query = text_embedding
2348+
else:
2349+
final_query = None
2350+
2351+
effective_query = final_query
2352+
2353+
assert effective_query is not None or query_string is not None, (
2354+
"Either query embedding or query_string must be provided."
22572355
)
22582356

22592357
(
@@ -2295,7 +2393,7 @@ def retrieve_online_documents_v2(
22952393
provider,
22962394
requested_feature_view,
22972395
requested_features,
2298-
query,
2396+
effective_query,
22992397
top_k,
23002398
distance_metric,
23012399
query_string,

0 commit comments

Comments
 (0)