From 5ce2e2454b1a2ff2ee466cd95782a47e665edec7 Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Wed, 14 Jun 2023 12:52:50 +0200 Subject: [PATCH 1/5] fix: fix runtime config --- docarray/index/backends/elastic.py | 96 ++++++++++++++-------------- docarray/index/backends/elasticv7.py | 22 ++++--- docarray/index/backends/hnswlib.py | 53 +++++++-------- docarray/index/backends/in_memory.py | 51 +++++++-------- docarray/index/backends/qdrant.py | 2 - 5 files changed, 113 insertions(+), 111 deletions(-) diff --git a/docarray/index/backends/elastic.py b/docarray/index/backends/elastic.py index 3a09de11527..45cb10f9203 100644 --- a/docarray/index/backends/elastic.py +++ b/docarray/index/backends/elastic.py @@ -98,7 +98,7 @@ def __init__(self, db_config=None, **kwargs): if issubclass(col.docarray_type, AnyDocArray): continue if col.db_type == 'dense_vector' and ( - not col.n_dim and col.config['dims'] < 0 + not col.n_dim and col.config['dims'] < 0 ): self._logger.info( f'Not indexing column {col_name}, the dimensionality is not specified' @@ -165,11 +165,11 @@ def build(self, *args, **kwargs) -> Any: return self._query def find( - self, - query: Union[AnyTensor, BaseDoc], - search_field: str = 'embedding', - limit: int = 10, - num_candidates: Optional[int] = None, + self, + query: Union[AnyTensor, BaseDoc], + search_field: str = 'embedding', + limit: int = 10, + num_candidates: Optional[int] = None, ): """ Find k-nearest neighbors of the query. @@ -365,10 +365,10 @@ def python_type_to_db_type(self, python_type: Type) -> Any: raise ValueError(err_msg) def _index( - self, - column_to_data: Mapping[str, Generator[Any, None, None]], - refresh: bool = True, - chunk_size: Optional[int] = None, + self, + column_to_data: Mapping[str, Generator[Any, None, None]], + refresh: bool = True, + chunk_size: Optional[int] = None, ): self._index_subindex(column_to_data) @@ -407,9 +407,9 @@ def num_docs(self) -> int: return self._client.count(index=self.index_name)['count'] def _del_items( - self, - doc_ids: Sequence[str], - chunk_size: Optional[int] = None, + self, + doc_ids: Sequence[str], + chunk_size: Optional[int] = None, ): requests = [] for _id in doc_ids: @@ -475,7 +475,7 @@ def execute_query(self, query: Dict[str, Any], *args, **kwargs) -> Any: return _FindResult(documents=docs, scores=parse_obj_as(NdArray, scores)) def _find( - self, query: np.ndarray, limit: int, search_field: str = '' + self, query: np.ndarray, limit: int, search_field: str = '' ) -> _FindResult: body = self._form_search_body(query, limit, search_field) @@ -486,10 +486,10 @@ def _find( return _FindResult(documents=docs, scores=parse_obj_as(NdArray, scores)) def _find_batched( - self, - queries: np.ndarray, - limit: int, - search_field: str = '', + self, + queries: np.ndarray, + limit: int, + search_field: str = '', ) -> _FindResultBatched: request = [] for query in queries: @@ -505,9 +505,9 @@ def _find_batched( return _FindResultBatched(documents=list(das), scores=scores) def _filter( - self, - filter_query: Dict[str, Any], - limit: int, + self, + filter_query: Dict[str, Any], + limit: int, ) -> List[Dict]: resp = self._client_search(query=filter_query, size=limit) @@ -516,9 +516,9 @@ def _filter( return docs def _filter_batched( - self, - filter_queries: Any, - limit: int, + self, + filter_queries: Any, + limit: int, ) -> List[List[Dict]]: request = [] for query in filter_queries: @@ -532,10 +532,10 @@ def _filter_batched( return list(das) def _text_search( - self, - query: str, - limit: int, - search_field: str = '', + self, + query: str, + limit: int, + search_field: str = '', ) -> _FindResult: body = self._form_text_search_body(query, limit, search_field) resp = self._client_search(**body) @@ -545,10 +545,10 @@ def _text_search( return _FindResult(documents=docs, scores=np.array(scores)) # type: ignore def _text_search_batched( - self, - queries: Sequence[str], - limit: int, - search_field: str = '', + self, + queries: Sequence[str], + limit: int, + search_field: str = '', ) -> _FindResultBatched: request = [] for query in queries: @@ -590,22 +590,22 @@ def _create_index_mapping(self, col: '_ColumnInfo') -> Dict[str, Any]: return index def _send_requests( - self, - request: Iterable[Dict[str, Any]], - chunk_size: Optional[int] = None, - **kwargs, + self, + request: Iterable[Dict[str, Any]], + chunk_size: Optional[int] = None, + **kwargs, ) -> Tuple[List[Dict], List[Any]]: """Send bulk request to Elastic and gather the successful info""" accumulated_info = [] warning_info = [] for success, info in parallel_bulk( - self._client, - request, - raise_on_error=False, - raise_on_exception=False, - chunk_size=chunk_size if chunk_size else self._runtime_config.chunk_size, # type: ignore - **kwargs, + self._client, + request, + raise_on_error=False, + raise_on_exception=False, + chunk_size=chunk_size if chunk_size else self._runtime_config.chunk_size, # type: ignore + **kwargs, ): if not success: warning_info.append(info) @@ -615,11 +615,11 @@ def _send_requests( return accumulated_info, warning_info def _form_search_body( - self, - query: np.ndarray, - limit: int, - search_field: str = '', - num_candidates: Optional[int] = None, + self, + query: np.ndarray, + limit: int, + search_field: str = '', + num_candidates: Optional[int] = None, ) -> Dict[str, Any]: if not num_candidates: num_candidates = self._runtime_config.default_column_config['dense_vector'][ @@ -637,7 +637,7 @@ def _form_search_body( return body def _form_text_search_body( - self, query: str, limit: int, search_field: str = '' + self, query: str, limit: int, search_field: str = '' ) -> Dict[str, Any]: body = { 'size': limit, diff --git a/docarray/index/backends/elasticv7.py b/docarray/index/backends/elasticv7.py index 8fe63682acd..83daca42c89 100644 --- a/docarray/index/backends/elasticv7.py +++ b/docarray/index/backends/elasticv7.py @@ -36,9 +36,9 @@ class QueryBuilder(ElasticDocIndex.QueryBuilder): def build(self, *args, **kwargs) -> Any: """Build the elastic search v7 query object.""" if ( - 'script_score' in self._query['query'] - and 'bool' in self._query['query'] - and len(self._query['query']['bool']) > 0 + 'script_score' in self._query['query'] + and 'bool' in self._query['query'] + and len(self._query['query']['bool']) > 0 ): self._query['query']['script_score']['query'] = {} self._query['query']['script_score']['query']['bool'] = self._query[ @@ -49,11 +49,11 @@ def build(self, *args, **kwargs) -> Any: return self._query def find( - self, - query: Union[AnyTensor, BaseDoc], - search_field: str = 'embedding', - limit: int = 10, - num_candidates: Optional[int] = None, + self, + query: Union[AnyTensor, BaseDoc], + search_field: str = 'embedding', + limit: int = 10, + num_candidates: Optional[int] = None, ): """ Find k-nearest neighbors of the query. @@ -96,6 +96,7 @@ def dense_vector_config(self): @dataclass class RuntimeConfig(ElasticDocIndex.RuntimeConfig): """Dataclass that contains all "dynamic" configurations of ElasticDocIndex.""" + pass ############################################### @@ -142,8 +143,9 @@ def _create_index_mapping(self, col: '_ColumnInfo') -> Dict[str, Any]: return index - def _form_search_body(self, query: np.ndarray, limit: int, search_field: str = '') -> Dict[ - str, Any]: # type: ignore + def _form_search_body( + self, query: np.ndarray, limit: int, search_field: str = '' + ) -> Dict[str, Any]: # type: ignore body = { 'size': limit, 'query': { diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 8abcfff98f7..deda886e3a6 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -104,7 +104,7 @@ def __init__(self, db_config=None, **kwargs): # non-tensor type; don't create an index continue if not load_existing and ( - (not col.n_dim and col.config['dim'] < 0) or not col.config['index'] + (not col.n_dim and col.config['dim'] < 0) or not col.config['index'] ): # tensor type, but don't index self._logger.info( @@ -185,6 +185,7 @@ class DBConfig(BaseDocIndex.DBConfig): @dataclass class RuntimeConfig(BaseDocIndex.RuntimeConfig): """Dataclass that contains all "dynamic" configurations of HnswDocumentIndex.""" + pass ############################################### @@ -206,9 +207,9 @@ def python_type_to_db_type(self, python_type: Type) -> Any: return None # all types allowed, but no db type needed def _index( - self, - column_to_data: Dict[str, Generator[Any, None, None]], - docs_validated: Sequence[BaseDoc] = [], + self, + column_to_data: Dict[str, Generator[Any, None, None]], + docs_validated: Sequence[BaseDoc] = [], ): self._index_subindex(column_to_data) @@ -284,10 +285,10 @@ def execute_query(self, query: List[Tuple[str, Dict]], *args, **kwargs) -> Any: return find_res def _find_batched( - self, - queries: np.ndarray, - limit: int, - search_field: str = '', + self, + queries: np.ndarray, + limit: int, + search_field: str = '', ) -> _FindResultBatched: if self.num_docs() == 0: return _FindResultBatched(documents=[], scores=[]) # type: ignore @@ -305,7 +306,7 @@ def _find_batched( return _FindResultBatched(documents=result_das, scores=distances) def _find( - self, query: np.ndarray, limit: int, search_field: str = '' + self, query: np.ndarray, limit: int, search_field: str = '' ) -> _FindResult: if self.num_docs() == 0: return _FindResult(documents=[], scores=[]) # type: ignore @@ -319,9 +320,9 @@ def _find( ) def _filter( - self, - filter_query: Any, - limit: int, + self, + filter_query: Any, + limit: int, ) -> DocList: raise NotImplementedError( f'{type(self)} does not support filter-only queries.' @@ -330,9 +331,9 @@ def _filter( ) def _filter_batched( - self, - filter_queries: Any, - limit: int, + self, + filter_queries: Any, + limit: int, ) -> List[DocList]: raise NotImplementedError( f'{type(self)} does not support filter-only queries.' @@ -341,25 +342,25 @@ def _filter_batched( ) def _text_search( - self, - query: str, - limit: int, - search_field: str = '', + self, + query: str, + limit: int, + search_field: str = '', ) -> _FindResult: raise NotImplementedError(f'{type(self)} does not support text search.') def _text_search_batched( - self, - queries: Sequence[str], - limit: int, - search_field: str = '', + self, + queries: Sequence[str], + limit: int, + search_field: str = '', ) -> _FindResultBatched: raise NotImplementedError(f'{type(self)} does not support text search.') def _del_items(self, doc_ids: Sequence[str]): # delete from the indices for field_name, type_, _ in self._flatten_schema( - cast(Type[BaseDoc], self._schema) + cast(Type[BaseDoc], self._schema) ): if safe_issubclass(type_, AnyDocArray): for id in doc_ids: @@ -423,7 +424,7 @@ def _to_hashed_id(doc_id: Optional[str]) -> int: raise ValueError( 'The Document id is None. To use DocumentIndex it needs to be set.' ) - return int(hashlib.sha256(doc_id.encode('utf-8')).hexdigest(), 16) % 10 ** 18 + return int(hashlib.sha256(doc_id.encode('utf-8')).hexdigest(), 16) % 10**18 def _load_index(self, col_name: str, col: '_ColumnInfo') -> hnswlib.Index: """Load an existing HNSW index from disk.""" @@ -479,7 +480,7 @@ def _get_docs_sqlite_unsorted(self, univ_ids: Sequence[int], out: bool = True): return docs_cls([self._doc_from_bytes(row[0], out) for row in rows]) def _get_docs_sqlite_doc_id( - self, doc_ids: Sequence[str], out: bool = True + self, doc_ids: Sequence[str], out: bool = True ) -> DocList[TSchema]: hashed_ids = tuple(self._to_hashed_id(id_) for id_ in doc_ids) docs_unsorted = self._get_docs_sqlite_unsorted(hashed_ids, out) diff --git a/docarray/index/backends/in_memory.py b/docarray/index/backends/in_memory.py index dc939ec31c6..3aa3f666924 100644 --- a/docarray/index/backends/in_memory.py +++ b/docarray/index/backends/in_memory.py @@ -46,10 +46,10 @@ class InMemoryExactNNIndex(BaseDocIndex, Generic[TSchema]): def __init__( - self, - docs: Optional[DocList] = None, - index_file_path: Optional[str] = None, - **kwargs, + self, + docs: Optional[DocList] = None, + index_file_path: Optional[str] = None, + **kwargs, ): """Initialize InMemoryExactNNIndex""" if 'db_config' in kwargs: @@ -149,6 +149,7 @@ class DBConfig(BaseDocIndex.DBConfig): @dataclass class RuntimeConfig(BaseDocIndex.RuntimeConfig): """Dataclass that contains all "dynamic" configurations of InMemoryExactNNIndex.""" + pass def index(self, docs: Union[BaseDoc, Sequence[BaseDoc]], **kwargs): @@ -235,7 +236,7 @@ def _ori_items(self, doc: BaseDoc) -> BaseDoc: ori_doc = _shallow_copy_doc(doc) for field_name, type_, _ in self._flatten_schema( - cast(Type[BaseDoc], self.out_schema) + cast(Type[BaseDoc], self.out_schema) ): if safe_issubclass(type_, AnyDocArray): _list = getattr(ori_doc, field_name) @@ -252,7 +253,7 @@ def _ori_items(self, doc: BaseDoc) -> BaseDoc: return ori_doc def _get_items( - self, doc_ids: Sequence[str], raw: bool = False + self, doc_ids: Sequence[str], raw: bool = False ) -> Union[Sequence[TSchema], Sequence[Dict[str, Any]]]: """Get Documents from the index, by `id`. If no document is found, a KeyError is raised. @@ -303,11 +304,11 @@ def execute_query(self, query: List[Tuple[str, Dict]], *args, **kwargs) -> Any: return find_res def find( - self, - query: Union[AnyTensor, BaseDoc], - search_field: str = '', - limit: int = 10, - **kwargs, + self, + query: Union[AnyTensor, BaseDoc], + search_field: str = '', + limit: int = 10, + **kwargs, ) -> FindResult: """Find Documents in the index using nearest-neighbor search. @@ -350,16 +351,16 @@ def find( return FindResult(documents=docs_with_schema, scores=scores) def _find( - self, query: np.ndarray, limit: int, search_field: str = '' + self, query: np.ndarray, limit: int, search_field: str = '' ) -> _FindResult: raise NotImplementedError def find_batched( - self, - queries: Union[AnyTensor, DocList], - search_field: str = '', - limit: int = 10, - **kwargs, + self, + queries: Union[AnyTensor, DocList], + search_field: str = '', + limit: int = 10, + **kwargs, ) -> FindResultBatched: """Find Documents in the index using nearest-neighbor search. @@ -393,15 +394,15 @@ def find_batched( return find_res def _find_batched( - self, queries: np.ndarray, limit: int, search_field: str = '' + self, queries: np.ndarray, limit: int, search_field: str = '' ) -> _FindResultBatched: raise NotImplementedError def filter( - self, - filter_query: Any, - limit: int = 10, - **kwargs, + self, + filter_query: Any, + limit: int = 10, + **kwargs, ) -> DocList: """Find documents in the index based on a filter query @@ -419,17 +420,17 @@ def _filter(self, filter_query: Any, limit: int) -> Union[DocList, List[Dict]]: raise NotImplementedError def _filter_batched( - self, filter_queries: Any, limit: int + self, filter_queries: Any, limit: int ) -> Union[List[DocList], List[List[Dict]]]: raise NotImplementedError(f'{type(self)} does not support filtering.') def _text_search( - self, query: str, limit: int, search_field: str = '' + self, query: str, limit: int, search_field: str = '' ) -> _FindResult: raise NotImplementedError(f'{type(self)} does not support text search.') def _text_search_batched( - self, queries: Sequence[str], limit: int, search_field: str = '' + self, queries: Sequence[str], limit: int, search_field: str = '' ) -> _FindResultBatched: raise NotImplementedError(f'{type(self)} does not support text search.') diff --git a/docarray/index/backends/qdrant.py b/docarray/index/backends/qdrant.py index 2c7b9343faa..45bc582d13a 100644 --- a/docarray/index/backends/qdrant.py +++ b/docarray/index/backends/qdrant.py @@ -255,8 +255,6 @@ class RuntimeConfig(BaseDocIndex.RuntimeConfig): pass - - def python_type_to_db_type(self, python_type: Type) -> Any: """Map python type to database type. Takes any python type and returns the corresponding database column type. From 0872d08a600bd2ff1535bf62c49c03d5856c9d4c Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Wed, 14 Jun 2023 16:37:25 +0200 Subject: [PATCH 2/5] test: fix tests --- docarray/index/backends/elastic.py | 2 +- docarray/index/backends/elasticv7.py | 4 +--- docarray/index/backends/weaviate.py | 9 ++++----- tests/index/base_classes/test_configs.py | 16 ++++++---------- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/docarray/index/backends/elastic.py b/docarray/index/backends/elastic.py index f7d6b265d3d..ad361414bbf 100644 --- a/docarray/index/backends/elastic.py +++ b/docarray/index/backends/elastic.py @@ -622,7 +622,7 @@ def _form_search_body( num_candidates: Optional[int] = None, ) -> Dict[str, Any]: if not num_candidates: - num_candidates = self._runtime_config.default_column_config['dense_vector'][ + num_candidates = self._db_config.default_column_config['dense_vector'][ 'num_candidates' ] body = { diff --git a/docarray/index/backends/elasticv7.py b/docarray/index/backends/elasticv7.py index 83daca42c89..db6e229bf25 100644 --- a/docarray/index/backends/elasticv7.py +++ b/docarray/index/backends/elasticv7.py @@ -143,9 +143,7 @@ def _create_index_mapping(self, col: '_ColumnInfo') -> Dict[str, Any]: return index - def _form_search_body( - self, query: np.ndarray, limit: int, search_field: str = '' - ) -> Dict[str, Any]: # type: ignore + def _form_search_body(self, query: np.ndarray, limit: int, search_field: str = '') -> Dict[str, Any]: # type: ignore body = { 'size': limit, 'query': { diff --git a/docarray/index/backends/weaviate.py b/docarray/index/backends/weaviate.py index 66a30f0ca46..39f626e9fd5 100644 --- a/docarray/index/backends/weaviate.py +++ b/docarray/index/backends/weaviate.py @@ -245,11 +245,6 @@ class DBConfig(BaseDocIndex.DBConfig): scopes: List[str] = field(default_factory=lambda: ["offline_access"]) auth_api_key: Optional[str] = None embedded_options: Optional[EmbeddedOptions] = None - - @dataclass - class RuntimeConfig(BaseDocIndex.RuntimeConfig): - """Dataclass that contains all "dynamic" configurations of WeaviateDocumentIndex.""" - default_column_config: Dict[Any, Dict[str, Any]] = field( default_factory=lambda: { np.ndarray: {}, @@ -264,6 +259,10 @@ class RuntimeConfig(BaseDocIndex.RuntimeConfig): } ) + @dataclass + class RuntimeConfig(BaseDocIndex.RuntimeConfig): + """Dataclass that contains all "dynamic" configurations of WeaviateDocumentIndex.""" + batch_config: Dict[str, Any] = field( default_factory=lambda: DEFAULT_BATCH_CONFIG ) diff --git a/tests/index/base_classes/test_configs.py b/tests/index/base_classes/test_configs.py index 0036baa46d2..7b7efbea596 100644 --- a/tests/index/base_classes/test_configs.py +++ b/tests/index/base_classes/test_configs.py @@ -67,7 +67,7 @@ def test_defaults(): index = DummyDocIndex[SimpleDoc]() assert index._db_config.other == 5 assert index._db_config.work_dir == '.' - assert index._runtime_config.default_column_config[str] == { + assert index._db_config.default_column_config[str] == { 'dim': 128, 'space': 'l2', } @@ -78,15 +78,13 @@ def test_set_by_class(): index = DummyDocIndex[SimpleDoc](DBConfig(work_dir='hi', other=10)) assert index._db_config.other == 10 assert index._db_config.work_dir == 'hi' - index.configure(RuntimeConfig(default_column_config={}, default_ef=10)) - assert index._runtime_config.default_column_config == {} + index.configure(RuntimeConfig(default_ef=10)) + assert index._runtime_config.default_ef == 10 # change only some settings index = DummyDocIndex[SimpleDoc](DBConfig(work_dir='hi')) assert index._db_config.other == 5 assert index._db_config.work_dir == 'hi' - index.configure(RuntimeConfig(default_column_config={})) - assert index._runtime_config.default_column_config == {} def test_set_by_kwargs(): @@ -94,20 +92,18 @@ def test_set_by_kwargs(): index = DummyDocIndex[SimpleDoc](work_dir='hi', other=10) assert index._db_config.other == 10 assert index._db_config.work_dir == 'hi' - index.configure(default_column_config={}, default_ef=10) - assert index._runtime_config.default_column_config == {} + index.configure(default_ef=10) + assert index._runtime_config.default_ef == 10 # change only some settings index = DummyDocIndex[SimpleDoc](work_dir='hi') assert index._db_config.other == 5 assert index._db_config.work_dir == 'hi' - index.configure(default_column_config={}) - assert index._runtime_config.default_column_config == {} def test_default_column_config(): index = DummyDocIndex[SimpleDoc]() - assert index._runtime_config.default_column_config == { + assert index._db_config.default_column_config == { str: { 'dim': 128, 'space': 'l2', From 34f02390b74cd56b5047d0c6fc6c6117336b49dc Mon Sep 17 00:00:00 2001 From: Joan Fontanals Martinez Date: Wed, 14 Jun 2023 17:10:48 +0200 Subject: [PATCH 3/5] docs: update docs --- docs/user_guide/storing/docindex.md | 11 +++-- docs/user_guide/storing/index_elastic.md | 19 ++++----- docs/user_guide/storing/index_hnswlib.md | 47 ++++++++++------------ docs/user_guide/storing/index_in_memory.md | 21 ++++------ 4 files changed, 44 insertions(+), 54 deletions(-) diff --git a/docs/user_guide/storing/docindex.md b/docs/user_guide/storing/docindex.md index 246bc7e0e30..fdd2e09f8aa 100644 --- a/docs/user_guide/storing/docindex.md +++ b/docs/user_guide/storing/docindex.md @@ -445,26 +445,25 @@ You can customize every field in this configuration: ### Runtime configurations -_Runtime configurations_ are configurations that pertain to the entire database or table (as opposed to just a specific column), -and that you can dynamically change at runtime. +_Runtime configurations_ are configurations that relate to the way how an `instance` operates with respect to a specific +database. This commonly includes: - default batch size for batching operations -- default mapping from pythong types to database column types - default consistency level for various database operations - ... For every backend, you can get the full list of configurations and their defaults: ```python -from docarray.index import HnswDocumentIndex +from docarray.index import ElasticDocIndex -runtime_config = HnswDocumentIndex.RuntimeConfig() +runtime_config = ElasticDocIndex.RuntimeConfig() print(runtime_config) -# > HnswDocumentIndex.RuntimeConfig(default_column_config={: {'dim': -1, 'index': True, 'space': 'l2', 'max_elements': 1024, 'ef_construction': 200, 'ef': 10, 'M': 16, 'allow_replace_deleted': True, 'num_threads': 1}, None: {}}) +# > ElasticDocIndex.RuntimeConfig(chunk_size=500) ``` As you can see, `HnswDocumentIndex.RuntimeConfig` is a dataclass that contains only one configuration: diff --git a/docs/user_guide/storing/index_elastic.md b/docs/user_guide/storing/index_elastic.md index 59ebdabeb82..b8251e9c88f 100644 --- a/docs/user_guide/storing/index_elastic.md +++ b/docs/user_guide/storing/index_elastic.md @@ -491,19 +491,11 @@ The following configs can be set in `DBConfig`: | `index_name` | Elasticsearch index name, the name of Elasticsearch index object | None. Data will be stored in an index named after the Document type used as schema. | | `index_settings` | Other [index settings](https://www.elastic.co/guide/en/elasticsearch/reference/8.6/index-modules.html#index-modules-settings) in a Dict for creating the index | dict | | `index_mappings` | Other [index mappings](https://www.elastic.co/guide/en/elasticsearch/reference/8.6/mapping.html) in a Dict for creating the index | dict | +| `default_column_config` | The default configurations for every column type. | dict | You can pass any of the above as keyword arguments to the `__init__()` method or pass an entire configuration object. See [here](docindex.md#configuration-options#customize-configurations) for more information. -### RuntimeConfig - -The `RuntimeConfig` dataclass of `ElasticDocIndex` consists of `default_column_config` and `chunk_size`. You can change `chunk_size` for batch operations: - -```python -doc_index = ElasticDocIndex[SimpleDoc]() -doc_index.configure(ElasticDocIndex.RuntimeConfig(chunk_size=1000)) -``` - `default_column_config` is the default configurations for every column type. Since there are many column types in Elasticsearch, you can also consider changing the column config when defining the schema. ```python @@ -514,5 +506,14 @@ class SimpleDoc(BaseDoc): doc_index = ElasticDocIndex[SimpleDoc]() ``` +### RuntimeConfig + +The `RuntimeConfig` dataclass of `ElasticDocIndex` consists of `chunk_size`. You can change `chunk_size` for batch operations: + +```python +doc_index = ElasticDocIndex[SimpleDoc]() +doc_index.configure(ElasticDocIndex.RuntimeConfig(chunk_size=1000)) +``` + You can pass the above as keyword arguments to the `configure()` method or pass an entire configuration object. See [here](docindex.md#configuration-options#customize-configurations) for more information. diff --git a/docs/user_guide/storing/index_hnswlib.md b/docs/user_guide/storing/index_hnswlib.md index 8f76b7f17c6..5fa0d67ad7a 100644 --- a/docs/user_guide/storing/index_hnswlib.md +++ b/docs/user_guide/storing/index_hnswlib.md @@ -31,10 +31,10 @@ This section lays out the configurations and options that are specific to [HnswD ### DBConfig -The `DBConfig` of [HnswDocumentIndex][docarray.index.backends.hnswlib.HnswDocumentIndex] expects only one argument: -`work_dir`. +The `DBConfig` of [HnswDocumentIndex][docarray.index.backends.hnswlib.HnswDocumentIndex] contains two argument: +`work_dir` and `default_column_configs` -This is the location where all of the Index's data will be stored, namely the various HNSWLib indexes and the SQLite database. +`work_dir` is the location where all of the Index's data will be stored, namely the various HNSWLib indexes and the SQLite database. You can pass this directly to the constructor: @@ -58,36 +58,31 @@ To load existing data, you can specify a directory that stores data from a previ Hnswlib uses a file lock to prevent multiple processes from accessing the same index at the same time. This means that if you try to open an index that is already open in another process, you will get an error. To avoid this, you can specify a different `work_dir` for each process. + +`default_column_configs` contains the default mapping from Python types to column configurations. -### RuntimeConfig - -The `RuntimeConfig` of [HnswDocumentIndex][docarray.index.backends.hnswlib.HnswDocumentIndex] contains only one entry: -the default mapping from Python types to column configurations. You can see in the [section below](#field-wise-configurations) how to override configurations for specific fields. -If you want to set configurations globally, i.e. for all vector fields in your documents, you can do that using `RuntimeConfig`: +If you want to set configurations globally, i.e. for all vector fields in your documents, you can do that using `DBConfig` or passing it at `__init__`: ```python import numpy as np -db = HnswDocumentIndex[MyDoc](work_dir='/tmp/my_db') - -db.configure( - default_column_config={ - np.ndarray: { - 'dim': -1, - 'index': True, - 'space': 'ip', - 'max_elements': 2048, - 'ef_construction': 100, - 'ef': 15, - 'M': 8, - 'allow_replace_deleted': True, - 'num_threads': 5, - }, - None: {}, - } -) + +db = HnswDocumentIndex[MyDoc](work_dir='/tmp/my_db', default_column_config={ + np.ndarray: { + 'dim': -1, + 'index': True, + 'space': 'ip', + 'max_elements': 2048, + 'ef_construction': 100, + 'ef': 15, + 'M': 8, + 'allow_replace_deleted': True, + 'num_threads': 5, + }, + None: {}, + }) ``` This will set the default configuration for all vector fields to the one specified in the example above. diff --git a/docs/user_guide/storing/index_in_memory.md b/docs/user_guide/storing/index_in_memory.md index f4bf1414cc8..a7b44720142 100644 --- a/docs/user_guide/storing/index_in_memory.md +++ b/docs/user_guide/storing/index_in_memory.md @@ -46,26 +46,21 @@ new_doc_index = InMemoryExactNNIndex[MyDoc](index_file_path='docs.bin') This section lays out the configurations and options that are specific to [InMemoryExactNNIndex][docarray.index.backends.in_memory.InMemoryExactNNIndex]. -### RuntimeConfig - -The `RuntimeConfig` of [InMemoryExactNNIndex][docarray.index.backends.in_memory.InMemoryExactNNIndex] contains only one entry: +The `DBConfig` of [InMemoryExactNNIndex][docarray.index.backends.in_memory.InMemoryExactNNIndex] contains only one entry: the default mapping from Python types to column configurations. You can see in the [section below](#field-wise-configurations) how to override configurations for specific fields. -If you want to set configurations globally, i.e. for all vector fields in your Documents, you can do that using `RuntimeConfig`: +If you want to set configurations globally, i.e. for all vector fields in your Documents, you can do that using `DBConfig` or passing it at `__init__`:: ```python from collections import defaultdict from docarray.typing import AbstractTensor - -index.configure( - default_column_config=defaultdict( - dict, - { - AbstractTensor: {'space': 'cosine_sim'}, - }, - ) -) +new_doc_index = InMemoryExactNNIndex[MyDoc](default_column_config=defaultdict( + dict, + { + AbstractTensor: {'space': 'cosine_sim'}, + }, + )) ``` This will set the default configuration for all vector fields to the one specified in the example above. From a74af43d1396b8b96931ae7f4afc738353c6fc56 Mon Sep 17 00:00:00 2001 From: Johannes Messner Date: Wed, 14 Jun 2023 17:52:23 +0200 Subject: [PATCH 4/5] docs: explain how to change default column configs Signed-off-by: Johannes Messner --- docs/user_guide/storing/docindex.md | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/storing/docindex.md b/docs/user_guide/storing/docindex.md index fdd2e09f8aa..af9488a11e3 100644 --- a/docs/user_guide/storing/docindex.md +++ b/docs/user_guide/storing/docindex.md @@ -558,7 +558,33 @@ The `HnswDocumentIndex` above contains two columns which are configured differen - `tens` has a dimensionality of `100`, can take up to `12` elements, and uses the `cosine` similarity space - `tens_two` has a dimensionality of `10`, and uses the `ip` similarity space, and an `M` hyperparameter of 4 -All configurations that are not explicitly set will be taken from the `default_column_config` of the `RuntimeConfig`. +All configurations that are not explicitly set will be taken from the `default_column_config` of the `DBConfig`. +You can modify these defaults in the following way: + +```python +import numpy as np +from pydantic import Field + +from docarray import BaseDoc +from docarray.index import HnswDocumentIndex +from docarray.typing import NdArray + + +class Schema(BaseDoc): + tens: NdArray[100] = Field(max_elements=12, space='cosine') + tens_two: NdArray[10] = Field(M=4, space='ip') + + +# create a DBConfig for your Document Index +conf = HnswDocumentIndex.DBConfig(work_dir='/tmp/my_db') +# update the default max_elements for np.ndarray columns +conf.default_column_config.get(np.ndarray).update(max_elements=2048) +# create Document Index +# tens has a max_elements of 12, specified in the schema +# tens_two has a max_elements of 2048, specified by the default in the DBConfig +db = HnswDocumentIndex[Schema](conf) +``` + For an explanation of the configurations that are tweaked in this example, see the `HnswDocumentIndex` [documentation](index_hnswlib.md). From 322a25ae617e4f072d1260a6dc462734309693fb Mon Sep 17 00:00:00 2001 From: Johannes Messner Date: Wed, 14 Jun 2023 17:58:46 +0200 Subject: [PATCH 5/5] docs: update how to add doc index backend Signed-off-by: Johannes Messner Signed-off-by: Joan Fontanals Martinez --- docs/how_to/add_doc_index.md | 25 ++++++++++--------- docs/user_guide/storing/index_hnswlib.md | 31 +++++++++++++----------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/docs/how_to/add_doc_index.md b/docs/how_to/add_doc_index.md index 37b2c74ac44..28eb5b5e128 100644 --- a/docs/how_to/add_doc_index.md +++ b/docs/how_to/add_doc_index.md @@ -288,12 +288,12 @@ To define what can be stored in them, and what the default values are, you need ```python @dataclass class DBConfig(BaseDocIndex.DBConfig): - ... + default_column_config: Dict[Type, Dict[str, Any]] = ... @dataclass class RuntimeConfig(BaseDocIndex.RuntimeConfig): - default_column_config: Dict[Type, Dict[str, Any]] = ... + ... ``` !!! note @@ -306,16 +306,8 @@ The `DBConfig` class defines the static configurations of your Document Index. These are configurations that are tied to the database (or library) running in the background, such as `host`, `port`, etc. Here you should put everything that the user cannot or should not change after initialization. -### The `RuntimeConfig` class - -The `RuntimeConfig` class defines the dynamic configurations of your Document Index. -These are configurations that can be changed at runtime, for example default behaviours such as batch sizes, consistency levels, etc. - -It is a common pattern to allow such parameters both in the `RuntimeConfig`, where they will act as global defaults, and -in specific methods (`index`, `find`, etc.), where they will act as local overrides. - !!! note - Every `RuntimeConfig` needs to contain a `default_column_config` field. + Every `DBConfig` needs to contain a `default_column_config` field. This is a dictionary that, for each possible column type in your database, defines a default configuration for that column type. This will automatically be passed to a `_ColumnInfo` whenever a user does not manually specify a configuration for that column. @@ -327,6 +319,15 @@ and for `varchar` columns you could define a `max_length` configuration. It is probably best to see this in action, so you should check out the `HnswDocumentIndex` implementation. +### The `RuntimeConfig` class + +The `RuntimeConfig` class defines the dynamic configurations of your Document Index. +These are configurations that can be changed at runtime, for example default behaviours such as batch sizes, consistency levels, etc. + +It is a common pattern to allow such parameters both in the `RuntimeConfig`, where they will act as global defaults, and +in specific methods (`index`, `find`, etc.), where they will act as local overrides. + + ## Implement abstract methods for indexing, searching, and deleting After you've done the basic setup above, you can jump into the good stuff: implementing the actual indexing, searching, and deleting. @@ -374,7 +375,7 @@ class MySchema(BaseDoc): In this case, the `db_type` of `my_num` will be `'float64'` and the `db_type` of `my_text` will be `'varchar'`. Additional information regarding the `col_type`, such as `max_len` for `varchar` will be stored in the `_ColumnsInfo.config`. -The given `col_type` has to be a valid `db_type`, meaning that has to be described in the index's `RuntimeConfig.default_column_config`. +The given `col_type` has to be a valid `db_type`, meaning that has to be described in the index's `DBConfig.default_column_config`. ### The `_index()` method diff --git a/docs/user_guide/storing/index_hnswlib.md b/docs/user_guide/storing/index_hnswlib.md index 5fa0d67ad7a..d19973389aa 100644 --- a/docs/user_guide/storing/index_hnswlib.md +++ b/docs/user_guide/storing/index_hnswlib.md @@ -69,20 +69,23 @@ If you want to set configurations globally, i.e. for all vector fields in your d import numpy as np -db = HnswDocumentIndex[MyDoc](work_dir='/tmp/my_db', default_column_config={ - np.ndarray: { - 'dim': -1, - 'index': True, - 'space': 'ip', - 'max_elements': 2048, - 'ef_construction': 100, - 'ef': 15, - 'M': 8, - 'allow_replace_deleted': True, - 'num_threads': 5, - }, - None: {}, - }) +db = HnswDocumentIndex[MyDoc]( + work_dir='/tmp/my_db', + default_column_config={ + np.ndarray: { + 'dim': -1, + 'index': True, + 'space': 'ip', + 'max_elements': 2048, + 'ef_construction': 100, + 'ef': 15, + 'M': 8, + 'allow_replace_deleted': True, + 'num_threads': 5, + }, + None: {}, + }, +) ``` This will set the default configuration for all vector fields to the one specified in the example above.