JoanFM · Jun 1, 2023 · May 31, 2023 · Jun 1, 2023 · Jun 1, 2023
diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py
@@ -162,13 +162,13 @@ def build(self, *args, **kwargs) -> Any:
 
     @dataclass
     class DBConfig(BaseDocIndex.DBConfig):
-        """Dataclass that contains all "static" configurations of WeaviateDocumentIndex."""
+        """Dataclass that contains all "static" configurations of HnswDocumentIndex."""
 
         work_dir: str = '.'
 
     @dataclass
     class RuntimeConfig(BaseDocIndex.RuntimeConfig):
-        """Dataclass that contains all "dynamic" configurations of WeaviateDocumentIndex."""
+        """Dataclass that contains all "dynamic" configurations of HnswDocumentIndex."""
 
         default_column_config: Dict[Type, Dict[str, Any]] = field(
             default_factory=lambda: {
@@ -221,6 +221,15 @@ def _index(
             data = column_to_data[col_name]
             data_np = [self._to_numpy(arr) for arr in data]
             data_stacked = np.stack(data_np)
+            num_docs_to_index = len(hashed_ids)
+            index_max_elements = index.get_max_elements()
+            current_elements = index.get_current_count()
+            if current_elements + num_docs_to_index > index_max_elements:
+                new_capacity = max(
+                    index_max_elements, current_elements + num_docs_to_index
+                )
+                self._logger.info(f'Resizing the index to {new_capacity}')
+                index.resize_index(new_capacity)
             index.add_items(data_stacked, ids=hashed_ids)
             index.save_index(self._hnsw_locations[col_name])
 
@@ -405,9 +414,7 @@ def _to_hashed_id(doc_id: Optional[str]) -> int:
     def _load_index(self, col_name: str, col: '_ColumnInfo') -> hnswlib.Index:
         """Load an existing HNSW index from disk."""
         index = self._create_index_class(col)
-        index.load_index(
-            self._hnsw_locations[col_name], max_elements=col.config['max_elements']
-        )
+        index.load_index(self._hnsw_locations[col_name])
         return index
 
     # HNSWLib helpers

diff --git a/docs/user_guide/storing/index_hnswlib.md b/docs/user_guide/storing/index_hnswlib.md
@@ -95,6 +95,11 @@ This will set the default configuration for all vector fields to the one specifi
 !!! note
     Even if your vectors come from PyTorch or TensorFlow, you can (and should) still use the `np.ndarray` configuration.
     This is because all tensors are converted to `np.ndarray` under the hood.
+
+!!! note
+   max_elements is considered to have the initial maximum capacity of the index. However, the capacity of the index is doubled every time
+   that the number of Documents in the index exceeds this capacity. Expanding the capacity is an expensive operation, therefore it can be important to
+   choose an appropiate max_elements value at init time.
 
 For more information on these settings, see [below](#field-wise-configurations).
 

diff --git a/tests/index/hnswlib/test_find.py b/tests/index/hnswlib/test_find.py
@@ -237,3 +237,58 @@ class MyDoc(BaseDoc):
     for q, matches in zip(queries, docs_responses):
         assert len(matches) == 10
         assert q.id == matches[0].id
+
+
+def test_usage_adapt_max_elements(tmpdir):
+    class MyDoc(BaseDoc):
+        text: str
+        embedding: NdArray[128]
+
+    docs = DocList[MyDoc](
+        [MyDoc(text='hey', embedding=np.random.rand(128)) for _ in range(200)]
+    )
+    queries = docs[0:3]
+    index = HnswDocumentIndex[MyDoc](work_dir=str(tmpdir))
+    index.configure()  # trying to configure the index but I am not managing to do so.
+    index.index(docs=docs)
+    resp = index.find_batched(queries=queries, search_field='embedding', limit=10)
+    docs_responses = resp.documents
+    assert len(docs_responses) == 3
+    for q, matches in zip(queries, docs_responses):
+        assert len(matches) == 10
+        assert q.id == matches[0].id
+
+
+def test_usage_adapt_max_elements_after_restore(tmpdir):
+    class MyDoc(BaseDoc):
+        text: str
+        embedding: NdArray[128]
+
+    docs = DocList[MyDoc](
+        [MyDoc(text='hey', embedding=np.random.rand(128)) for _ in range(200)]
+    )
+    queries = docs[0:3]
+    index = HnswDocumentIndex[MyDoc](work_dir=str(tmpdir))
+    index.configure()  # trying to configure the index but I am not managing to do so.
+    index.index(docs=docs)
+    resp = index.find_batched(queries=queries, search_field='embedding', limit=10)
+    docs_responses = resp.documents
+    assert len(docs_responses) == 3
+    for q, matches in zip(queries, docs_responses):
+        assert len(matches) == 10
+        assert q.id == matches[0].id
+
+    new_docs = DocList[MyDoc](
+        [MyDoc(text='hey', embedding=np.random.rand(128)) for _ in range(200)]
+    )
+    restored_index = HnswDocumentIndex[MyDoc](work_dir=str(tmpdir))
+    restored_index.index(docs=new_docs)
+    queries = new_docs[0:3]
+    resp = restored_index.find_batched(
+        queries=queries, search_field='embedding', limit=10
+    )
+    docs_responses = resp.documents
+    assert len(docs_responses) == 3
+    for q, matches in zip(queries, docs_responses):
+        assert len(matches) == 10
+        assert q.id == matches[0].id