Skip to content

feat: Adding Milvus demo to examples #4910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
franciscojavierarceo merged 17 commits into from
Jan 30, 2025
Merged

feat: Adding Milvus demo to examples #4910

Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
468368a
checking in progress but this Pr still is not ready yet
franciscojavierarceo Jan 25, 2025
8f26494
feat: Adding new method to FeatureStore to allow more flexible retrie…
franciscojavierarceo Jan 27, 2025
ddc963b
Adding requested_features back into online_store
franciscojavierarceo Jan 27, 2025
006e897
feat: Adding RAG demo displaying Milvus usage for RAG
franciscojavierarceo Jan 8, 2025
a1a9408
uploading sample data and updated yaml
franciscojavierarceo Jan 10, 2025
4fba66c
updating workflow
franciscojavierarceo Jan 10, 2025
c4736b7
updated example
franciscojavierarceo Jan 10, 2025
f7393a7
removing modified files
franciscojavierarceo Jan 10, 2025
d0a1e30
reverting postgres change
franciscojavierarceo Jan 10, 2025
0d4c351
updating test_workflow
franciscojavierarceo Jan 13, 2025
7c006b1
updated and fixed bug
franciscojavierarceo Jan 28, 2025
63aa43b
fixing a bad merge/rebase
franciscojavierarceo Jan 29, 2025
aa81350
updated linter because of latest nistall
franciscojavierarceo Jan 29, 2025
e2a1949
reverting feature store change
franciscojavierarceo Jan 29, 2025
017ff54
adding logging of local milvus back
franciscojavierarceo Jan 29, 2025
ac96e36
Updating readme and adding notebook
franciscojavierarceo Jan 29, 2025
c629a42
updated readme
franciscojavierarceo Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
updating workflow 
Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>
  • Loading branch information
@franciscojavierarceo
franciscojavierarceo committed Jan 29, 2025
commit 4fba66c2e2cf301d91cc9317052b68f5ddfd2e5a
44 changes: 38 additions & 6 deletions examples/rag/feature_repo/test_workflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import torch
import torch.nn.functional as F
from feast import FeatureStore
from pymilvus import MilvusClient
from pymilvus import MilvusClient, DataType, FieldSchema
from transformers import AutoTokenizer, AutoModel
from example_repo import city_embeddings_feature_view, item
TOKENIZER = "sentence-transformers/all-MiniLM-L6-v2"
Expand Down Expand Up @@ -36,12 +36,43 @@ def run_model(sentences, tokenizer, model):
def run_demo():
store = FeatureStore(repo_path=".")
df = pd.read_parquet("./data/city_wikipedia_summaries_with_embeddings.parquet")
store.apply([city_embeddings_feature_view, item])
store.write_to_online_store_async("city_embeddings", df)
embedding_length = len(df['vector'][0])
print(f'embedding length = {embedding_length}')

print('\ndata=')
print(df.head().T)

client = MilvusClient(alias="feast", host="localhost", port="19530", token="username:password")
print(client.list_collections())
store.apply([city_embeddings_feature_view, item])
store.write_to_online_store("city_embeddings", df)

client = MilvusClient(uir="http://localhost:19530", token="username:password")
fields = [
FieldSchema(name="id", dtype=DataType.INT64, is_primary=True),
FieldSchema(name='state', dtype=DataType.STRING, description="State"),
FieldSchema(name='wiki_summary', dtype=DataType.STRING, description="State"),
FieldSchema(name='sentence_chunks', dtype=DataType.STRING, description="Sentence Chunks"),
FieldSchema(name="item_id", dtype=DataType.INT64, default_value=0, description="Item"),
FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=embedding_length, description="vector")
]
cols = [f.name for f in fields]
client.insert(
collection_name="demo_collection",
data=df[cols].to_dict(orient="records"),
schema=fields,
)
print('\n')
print('collections', client.list_collections())
print('query results =', client.query(
collection_name="rag_city_embeddings",
filter="item_id == 0",
# output_fields=['city_embeddings', 'item_id', 'city_name'],
))
print('query results2 =', client.query(
collection_name="rag_city_embeddings",
filter="item_id >= 0",
output_fields=["count(*)"]
# output_fields=['city_embeddings', 'item_id', 'city_name'],
))
question = "the most populous city in the U.S. state of Texas?"
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER)
model = AutoModel.from_pretrained(MODEL)
Expand All @@ -50,7 +81,8 @@ def run_demo():

# Retrieve top k documents
features = store.retrieve_online_documents(
feature="city_embeddings:Embeddings",
feature=None,
features=["city_embeddings:vector", "city_embeddings:item_id", "city_embeddings:state"],
query=query,
top_k=3
)
Expand Down