From 86cc22aa30de029b95fe3c162a387793f0eb1a40 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Wed, 31 May 2023 18:32:07 +0200 Subject: [PATCH 1/4] fix: allow update on hnswlib Signed-off-by: Joan Fontanals Martinez --- docarray/index/backends/hnswlib.py | 2 +- docs/user_guide/storing/index_hnswlib.md | 5 ++- tests/index/hnswlib/test_index_get_del.py | 50 +++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 5d7f951fe9f..44033a7bace 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -439,7 +439,7 @@ def _create_docs_table(self): def _send_docs_to_sqlite(self, docs: Sequence[BaseDoc]): ids = (self._to_hashed_id(doc.id) for doc in docs) self._sqlite_cursor.executemany( - 'INSERT INTO docs VALUES (?, ?)', + 'INSERT OR REPLACE INTO docs VALUES (?, ?)', ((id_, self._doc_to_bytes(doc)) for id_, doc in zip(ids, docs)), ) diff --git a/docs/user_guide/storing/index_hnswlib.md b/docs/user_guide/storing/index_hnswlib.md index e05a1ff0c9f..6beae4e46f7 100644 --- a/docs/user_guide/storing/index_hnswlib.md +++ b/docs/user_guide/storing/index_hnswlib.md @@ -208,4 +208,7 @@ To delete nested data, you need to specify the `id`. del doc_index[index_docs[6].id] ``` -Check [here](docindex#nested-data-with-subindex) for nested data with subindex. \ No newline at end of file +Check [here](docindex#nested-data-with-subindex) for nested data with subindex. + +### Update elements +In order to update a Document inside the index, you only need to reindex it with the updated attributes. \ No newline at end of file diff --git a/tests/index/hnswlib/test_index_get_del.py b/tests/index/hnswlib/test_index_get_del.py index 4ecb60e465a..ebf06ce4b14 100644 --- a/tests/index/hnswlib/test_index_get_del.py +++ b/tests/index/hnswlib/test_index_get_del.py @@ -362,3 +362,53 @@ def test_num_docs(ten_simple_docs, tmp_path): del index[more_docs[2].id, ten_simple_docs[7].id] assert index.num_docs() == 10 + + +def test_update_payload(tmp_path): + class TextSimpleDoc(SimpleDoc): + text: str = 'hey' + + docs = DocList[TextSimpleDoc]( + [TextSimpleDoc(tens=np.random.rand(10), text=f'hey {i}') for i in range(100)] + ) + index = HnswDocumentIndex[TextSimpleDoc](work_dir=str(tmp_path)) + index.index(docs) + assert index.num_docs() == 100 + + for doc in docs: + doc.text += '_changed' + + index.index(docs) + assert index.num_docs() == 100 + + res = index.find(query=docs[0], search_field='tens', limit=100) + assert len(res.documents) == 100 + for doc in res.documents: + assert '_changed' in doc.text + + +def test_update_embedding(tmp_path): + class TextSimpleDoc(SimpleDoc): + text: str = 'hey' + + docs = DocList[TextSimpleDoc]( + [TextSimpleDoc(tens=np.random.rand(10), text=f'hey {i}') for i in range(100)] + ) + index = HnswDocumentIndex[TextSimpleDoc](work_dir=str(tmp_path)) + index.index(docs) + assert index.num_docs() == 100 + + new_tensor = np.random.rand(10) + docs[0].tens = new_tensor + + index.index(docs[0]) + assert index.num_docs() == 100 + + res = index.find(query=docs[0], search_field='tens', limit=100) + assert len(res.documents) == 100 + found = False + for doc in res.documents: + if doc.id == docs[0].id: + found = True + assert (doc.tens == new_tensor).all() + assert found From b2a92b53eb6f28f659c4ad8ef2052949a94af53e Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Thu, 1 Jun 2023 09:19:18 +0200 Subject: [PATCH 2/4] docs: add code snippet in documentation Signed-off-by: Joan Fontanals Martinez --- docs/user_guide/storing/index_hnswlib.md | 36 +++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/storing/index_hnswlib.md b/docs/user_guide/storing/index_hnswlib.md index 6beae4e46f7..ce58430c78c 100644 --- a/docs/user_guide/storing/index_hnswlib.md +++ b/docs/user_guide/storing/index_hnswlib.md @@ -211,4 +211,38 @@ del doc_index[index_docs[6].id] Check [here](docindex#nested-data-with-subindex) for nested data with subindex. ### Update elements -In order to update a Document inside the index, you only need to reindex it with the updated attributes. \ No newline at end of file +In order to update a Document inside the index, you only need to reindex it with the updated attributes. + +```python +import numpy as np + +from docarray import BaseDoc, DocList +from docarray.typing import NdArray +from docarray.index import HnswDocumentIndex + +class MyDoc(BaseDoc): + text: str + embedding: NdArray[128] + +docs = DocList[MyDoc]( + [MyDoc(embedding=np.random.rand(10), text=f'I am the first version of Document {i}') for i in range(100)] +) +index = HnswDocumentIndex[MyDoc]() +index.index(docs) +assert index.num_docs() == 100 +res = index.find(query=docs[0], search_field='tens', limit=100) +assert len(res.documents) == 100 +for doc in res.documents: + assert 'I am the first version' in doc.text + +for i, doc in enumerate(docs): + doc.text = f'I am the second version of Document {i}' + +index.index(docs) +assert index.num_docs() == 100 + +res = index.find(query=docs[0], search_field='tens', limit=100) +assert len(res.documents) == 100 +for doc in res.documents: + assert 'I am the second version' in doc.text +``` From fa36ca776cdfa4feb47ac9abba9c18b49a36db96 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Thu, 1 Jun 2023 15:36:03 +0200 Subject: [PATCH 3/4] docs: update link Signed-off-by: Joan Fontanals Martinez --- docs/user_guide/storing/index_hnswlib.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guide/storing/index_hnswlib.md b/docs/user_guide/storing/index_hnswlib.md index ce58430c78c..e599eebba31 100644 --- a/docs/user_guide/storing/index_hnswlib.md +++ b/docs/user_guide/storing/index_hnswlib.md @@ -208,7 +208,7 @@ To delete nested data, you need to specify the `id`. del doc_index[index_docs[6].id] ``` -Check [here](docindex#nested-data-with-subindex) for nested data with subindex. +Check [here](../docindex#nested-data-with-subindex) for nested data with subindex. ### Update elements In order to update a Document inside the index, you only need to reindex it with the updated attributes. From f4e374c3d0049b89bc118a79e7c15c856b5df6fb Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Thu, 1 Jun 2023 16:14:51 +0200 Subject: [PATCH 4/4] docs: update docs as per comments --- docs/user_guide/storing/index_hnswlib.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/storing/index_hnswlib.md b/docs/user_guide/storing/index_hnswlib.md index e599eebba31..b85d903852d 100644 --- a/docs/user_guide/storing/index_hnswlib.md +++ b/docs/user_guide/storing/index_hnswlib.md @@ -213,34 +213,49 @@ Check [here](../docindex#nested-data-with-subindex) for nested data with subinde ### Update elements In order to update a Document inside the index, you only need to reindex it with the updated attributes. +First lets create a schema for our Index ```python import numpy as np - from docarray import BaseDoc, DocList from docarray.typing import NdArray from docarray.index import HnswDocumentIndex - class MyDoc(BaseDoc): text: str embedding: NdArray[128] +``` +Now we can instantiate our Index and index some data. +```python docs = DocList[MyDoc]( [MyDoc(embedding=np.random.rand(10), text=f'I am the first version of Document {i}') for i in range(100)] ) index = HnswDocumentIndex[MyDoc]() index.index(docs) assert index.num_docs() == 100 +``` + +Now we can find relevant documents + +```python res = index.find(query=docs[0], search_field='tens', limit=100) assert len(res.documents) == 100 for doc in res.documents: assert 'I am the first version' in doc.text +``` +and update all of the text of this documents and reindex them + +```python for i, doc in enumerate(docs): doc.text = f'I am the second version of Document {i}' index.index(docs) assert index.num_docs() == 100 +``` + +When we retrieve them again we can see that their text attribute has been updated accordingly +```python res = index.find(query=docs[0], search_field='tens', limit=100) assert len(res.documents) == 100 for doc in res.documents: