From bf6382e1b42766016f92df46065eef05cdaa4f03 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:12:51 +0530 Subject: [PATCH 01/16] solved the good first issue in the hnswlib.py file --- docarray/index/backends/hnswlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index c0ee904fb4..2dfff68428 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -428,7 +428,7 @@ def num_docs(self) -> int: Get the number of documents. """ if self._num_docs == 0: - self._num_docs = self._get_num_docs_sqlite() + self._num_docs = self._get_num_docs_hnsw() return self._num_docs ############################################### From 94663899c7720d24754abbf00ec91b365cfd6048 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Thu, 24 Aug 2023 21:05:52 +0530 Subject: [PATCH 02/16] Edited the code. --- docarray/index/backends/hnswlib.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 2dfff68428..fe7ba3bf53 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -430,7 +430,13 @@ def num_docs(self) -> int: if self._num_docs == 0: self._num_docs = self._get_num_docs_hnsw() return self._num_docs - + def _get_num_docs_hnsw(self) -> int: + """ + Get the number of documents using the HNSW method. + """ + # Access your HNSW index here and retrieve the count of documents + hnsw_num_docs = self.hnsw_index.get_document_count() + return hnsw_num_docs ############################################### # Helpers # ############################################### From 83fa2b7c9ddee1df1d9b65e068295dd40503bdfe Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Fri, 25 Aug 2023 05:33:47 +0530 Subject: [PATCH 03/16] fix: #1764, assigned the first indices Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index fe7ba3bf53..d9c9461109 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -32,9 +32,7 @@ _raise_not_composable, _raise_not_supported, ) -from docarray.index.backends.helper import ( - _collect_query_args, -) +from docarray.index.backends.helper import _collect_query_args from docarray.proto import DocProto from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.ndarray import NdArray @@ -430,13 +428,18 @@ def num_docs(self) -> int: if self._num_docs == 0: self._num_docs = self._get_num_docs_hnsw() return self._num_docs + def _get_num_docs_hnsw(self) -> int: """ Get the number of documents using the HNSW method. """ - # Access your HNSW index here and retrieve the count of documents - hnsw_num_docs = self.hnsw_index.get_document_count() + # Access the first HNSW index from self._hnsw_indices + first_hnsw_index = self._hnsw_indices[0] + hnsw_num_docs = ( + first_hnsw_index.get_document_count() + ) # Replace with actual method return hnsw_num_docs + ############################################### # Helpers # ############################################### From 45f4ae4946b1326e0bcebf689fb9f4f432065617 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:08:32 +0530 Subject: [PATCH 04/16] fix : #1764 Fix accessing HNSW index document count Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index d9c9461109..05aebe60d6 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -433,8 +433,8 @@ def _get_num_docs_hnsw(self) -> int: """ Get the number of documents using the HNSW method. """ - # Access the first HNSW index from self._hnsw_indices - first_hnsw_index = self._hnsw_indices[0] + first_index_key = next(iter(self._hnsw_indices)) # Get the first key + first_hnsw_index = self._hnsw_indices[first_index_key] hnsw_num_docs = ( first_hnsw_index.get_document_count() ) # Replace with actual method From 042d5c3869e23583887cea24bc884aeecf51a1bf Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Fri, 25 Aug 2023 22:06:58 +0530 Subject: [PATCH 05/16] fix: #1764 removed the comment Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 05aebe60d6..f3a80f6da3 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -435,9 +435,7 @@ def _get_num_docs_hnsw(self) -> int: """ first_index_key = next(iter(self._hnsw_indices)) # Get the first key first_hnsw_index = self._hnsw_indices[first_index_key] - hnsw_num_docs = ( - first_hnsw_index.get_document_count() - ) # Replace with actual method + hnsw_num_docs = first_hnsw_index.get_document_count() return hnsw_num_docs ############################################### From 8ecffda2eaf6b42d192d3ac5d564aa8b358612a6 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Mon, 28 Aug 2023 17:04:09 +0530 Subject: [PATCH 06/16] fixes: #1765, implementing element_count() Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index f3a80f6da3..8db63ccd43 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -435,7 +435,7 @@ def _get_num_docs_hnsw(self) -> int: """ first_index_key = next(iter(self._hnsw_indices)) # Get the first key first_hnsw_index = self._hnsw_indices[first_index_key] - hnsw_num_docs = first_hnsw_index.get_document_count() + hnsw_num_docs = first_hnsw_index.element_count() return hnsw_num_docs ############################################### From 13a9ebf7f832075644ab41d11434dd7a547a962d Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Mon, 28 Aug 2023 22:58:32 +0530 Subject: [PATCH 07/16] fixes: #1765 added the attribute(in the hnswlib.py) Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 8db63ccd43..ec03c30928 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -435,7 +435,7 @@ def _get_num_docs_hnsw(self) -> int: """ first_index_key = next(iter(self._hnsw_indices)) # Get the first key first_hnsw_index = self._hnsw_indices[first_index_key] - hnsw_num_docs = first_hnsw_index.element_count() + hnsw_num_docs = first_hnsw_index.element_count return hnsw_num_docs ############################################### From 0b428c64f78e9cec758b5e2407de27fc3a879434 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Sun, 3 Sep 2023 12:29:56 +0530 Subject: [PATCH 08/16] fixes #1765 Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index ec03c30928..c4bda4deb6 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -438,6 +438,14 @@ def _get_num_docs_hnsw(self) -> int: hnsw_num_docs = first_hnsw_index.element_count return hnsw_num_docs + def _count_docs_in_hnswlib(self) -> int: + # Use HNSWlib's method to count documents + return self._hnsw_indices.element_count + + def is_index_empty(self) -> bool: + # Check if the index is empty by counting documents in HNSWlib + return self._count_docs_in_hnswlib() == 0 + ############################################### # Helpers # ############################################### From 803fc5db5705852c5ec8e3a214612e70e4300b0a Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Mon, 4 Sep 2023 05:40:35 +0530 Subject: [PATCH 09/16] fixes: 1765 got the first index in the dictionary Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index c4bda4deb6..7ac52a06d0 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -429,18 +429,26 @@ def num_docs(self) -> int: self._num_docs = self._get_num_docs_hnsw() return self._num_docs - def _get_num_docs_hnsw(self) -> int: + # def _get_num_docs_hnsw(self) -> int: + # """ + # Get the number of documents using the HNSW method. + # """ + # first_index_key = next(iter(self._hnsw_indices)) # Get the first key + # first_hnsw_index = self._hnsw_indices[first_index_key] + # hnsw_num_docs = first_hnsw_index.element_count + # return hnsw_num_docs + + def _count_docs_in_hnswlib(self) -> int: """ - Get the number of documents using the HNSW method. + Get the number of documents using the HNSW method from the first index in the dictionary. """ - first_index_key = next(iter(self._hnsw_indices)) # Get the first key + # Get the first index from the dictionary + first_index_key = next(iter(self._hnsw_indices)) first_hnsw_index = self._hnsw_indices[first_index_key] - hnsw_num_docs = first_hnsw_index.element_count - return hnsw_num_docs - def _count_docs_in_hnswlib(self) -> int: # Use HNSWlib's method to count documents - return self._hnsw_indices.element_count + hnsw_num_docs = first_hnsw_index.element_count + return hnsw_num_docs def is_index_empty(self) -> bool: # Check if the index is empty by counting documents in HNSWlib From 1057805aec588fdcc264c1a582dfcc0fd638b3f8 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:48:30 +0530 Subject: [PATCH 10/16] fixes: #1765, step 1 Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 7ac52a06d0..0218be9322 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -429,30 +429,13 @@ def num_docs(self) -> int: self._num_docs = self._get_num_docs_hnsw() return self._num_docs - # def _get_num_docs_hnsw(self) -> int: - # """ - # Get the number of documents using the HNSW method. - # """ - # first_index_key = next(iter(self._hnsw_indices)) # Get the first key - # first_hnsw_index = self._hnsw_indices[first_index_key] - # hnsw_num_docs = first_hnsw_index.element_count - # return hnsw_num_docs - - def _count_docs_in_hnswlib(self) -> int: - """ - Get the number of documents using the HNSW method from the first index in the dictionary. + def is_index_empty(self) -> bool: """ - # Get the first index from the dictionary - first_index_key = next(iter(self._hnsw_indices)) - first_hnsw_index = self._hnsw_indices[first_index_key] - - # Use HNSWlib's method to count documents - hnsw_num_docs = first_hnsw_index.element_count - return hnsw_num_docs + Check if the index is empty by comparing the number of documents to zero. - def is_index_empty(self) -> bool: - # Check if the index is empty by counting documents in HNSWlib - return self._count_docs_in_hnswlib() == 0 + :return: True if the index is empty, False otherwise. + """ + return self.num_docs() == 0 ############################################### # Helpers # From f12bf2087d0868b4806a82c6d0429c01709355fd Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:46:31 +0530 Subject: [PATCH 11/16] fixes: #1765 step 1 again Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 0218be9322..5a85ca2996 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -687,7 +687,7 @@ def accept_hashed_ids(id): try: labels, distances = index.knn_query(queries, k=k, **extra_kwargs) except RuntimeError: - k = min(k, self.num_docs()) + k = min(k, self.is_index_empty()) labels, distances = index.knn_query(queries, k=k, **extra_kwargs) result_das = [ @@ -779,8 +779,8 @@ def _execute_find_and_filter_query( post_filters: Dict[str, Dict] = {} # Define filter limits - pre_filter_limit = self.num_docs() - post_filter_limit = self.num_docs() + pre_filter_limit = self.is_index_empty() + post_filter_limit = self.is_index_empty() find_executed: bool = False @@ -799,7 +799,7 @@ def _execute_find_and_filter_query( # Returns batched output, so we need to get the first lists out_docs, scores = self._search_and_filter( # type: ignore[assignment] queries=query_vector, - limit=op_kwargs.get('limit', self.num_docs()), + limit=op_kwargs.get('limit', self.is_index_empty()), search_field=op_kwargs['search_field'], hashed_ids=hashed_ids, ) @@ -895,7 +895,7 @@ def _post_filtering( :return: Filtered documents as per the post-filter conditions. """ if not find_executed: - out_docs = self.filter(post_filters, limit=self.num_docs()) + out_docs = self.filter(post_filters, limit=self.is_index_empty()) else: docs_cls = DocList.__class_getitem__(cast(Type[BaseDoc], self.out_schema)) out_docs = docs_cls(filter_docs(out_docs, post_filters)) From 81988c15df5477ea62e8ecb204192e2b8bb45c35 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:19:11 +0530 Subject: [PATCH 12/16] fixes: #1765 --- docarray/index/abstract.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 5b441316d5..52244b473e 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -115,6 +115,14 @@ class _NewSchema(self._schema): # type: ignore self._subindices: Dict[str, BaseDocIndex] = {} self._init_subindex() + def is_index_empty(self) -> bool: + """ + Check if the index is empty by comparing the number of documents to zero. + + :return: True if the index is empty, False otherwise. + """ + return self.num_docs() == 0 + ############################################### # Inner classes for query builder and configs # # Subclasses must subclass & implement these # From d576e60428247036c603095f0a3be49cfbe672a3 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:21:00 +0530 Subject: [PATCH 13/16] Revert "fixes: #1765 step 1 again" This reverts commit f12bf2087d0868b4806a82c6d0429c01709355fd. --- docarray/index/backends/hnswlib.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 5a85ca2996..0218be9322 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -687,7 +687,7 @@ def accept_hashed_ids(id): try: labels, distances = index.knn_query(queries, k=k, **extra_kwargs) except RuntimeError: - k = min(k, self.is_index_empty()) + k = min(k, self.num_docs()) labels, distances = index.knn_query(queries, k=k, **extra_kwargs) result_das = [ @@ -779,8 +779,8 @@ def _execute_find_and_filter_query( post_filters: Dict[str, Dict] = {} # Define filter limits - pre_filter_limit = self.is_index_empty() - post_filter_limit = self.is_index_empty() + pre_filter_limit = self.num_docs() + post_filter_limit = self.num_docs() find_executed: bool = False @@ -799,7 +799,7 @@ def _execute_find_and_filter_query( # Returns batched output, so we need to get the first lists out_docs, scores = self._search_and_filter( # type: ignore[assignment] queries=query_vector, - limit=op_kwargs.get('limit', self.is_index_empty()), + limit=op_kwargs.get('limit', self.num_docs()), search_field=op_kwargs['search_field'], hashed_ids=hashed_ids, ) @@ -895,7 +895,7 @@ def _post_filtering( :return: Filtered documents as per the post-filter conditions. """ if not find_executed: - out_docs = self.filter(post_filters, limit=self.is_index_empty()) + out_docs = self.filter(post_filters, limit=self.num_docs()) else: docs_cls = DocList.__class_getitem__(cast(Type[BaseDoc], self.out_schema)) out_docs = docs_cls(filter_docs(out_docs, post_filters)) From 229d73f2dca3ceddd9162511a43f0776f7c4ef40 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Tue, 19 Sep 2023 17:24:23 +0530 Subject: [PATCH 14/16] fixes: #1765 Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 52244b473e..f6e4fdbdaa 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -117,7 +117,7 @@ class _NewSchema(self._schema): # type: ignore def is_index_empty(self) -> bool: """ - Check if the index is empty by comparing the number of documents to zero. + Check if index is empty by comparing the number of documents to zero. :return: True if the index is empty, False otherwise. """ From 678f1dd27723aa94860f0ca0a09e6372011bf605 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Wed, 20 Sep 2023 18:30:51 +0530 Subject: [PATCH 15/16] fixes: #1765 --- docarray/index/backends/hnswlib.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 0218be9322..26212c087b 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -425,17 +425,9 @@ def num_docs(self) -> int: """ Get the number of documents. """ - if self._num_docs == 0: - self._num_docs = self._get_num_docs_hnsw() - return self._num_docs - - def is_index_empty(self) -> bool: - """ - Check if the index is empty by comparing the number of documents to zero. - - :return: True if the index is empty, False otherwise. - """ - return self.num_docs() == 0 + if self.is_index_empty == 0: + self.is_index_empty = self._get_num_docs_hnsw() + return self.is_index_empty ############################################### # Helpers # From 436937b7b336c80762fd620e7658fdfea972e5f0 Mon Sep 17 00:00:00 2001 From: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> Date: Wed, 20 Sep 2023 18:32:26 +0530 Subject: [PATCH 16/16] fixes: #1765 Signed-off-by: Shobhit Kumar <116124866+shobhit9957@users.noreply.github.com> --- docarray/index/backends/hnswlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 26212c087b..d5cf86e76f 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -423,7 +423,7 @@ def _doc_exists(self, doc_id: str) -> bool: def num_docs(self) -> int: """ - Get the number of documents. + Get number of documents. """ if self.is_index_empty == 0: self.is_index_empty = self._get_num_docs_hnsw()