From 25d6992466a577f2e44029c1038514afd44721c8 Mon Sep 17 00:00:00 2001 From: maxwelljin2 Date: Wed, 24 May 2023 18:25:13 +0800 Subject: [PATCH 1/7] Fix: Add a Custom issubclass Function to Handle Non-Class Inputs --- docarray/index/abstract.py | 28 ++++++++++++++++++++++------ docarray/index/backends/hnswlib.py | 6 +++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index d09f49ce8cb..3cd44f81db1 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -18,6 +18,7 @@ TypeVar, Union, cast, + get_origin, ) import numpy as np @@ -706,6 +707,21 @@ def _filter_by_parent_id(self, id: str) -> Optional[List[str]]: # These might be useful in your subclass implementation # ########################################################## + @staticmethod + def custom_issubclass(x, A_tuple) -> bool: + """ + This is a modified version of the built-in 'issubclass' function to support non-class input. + Traditional 'issubclass' calls can result in a crash if the input is list/tuple. + + :param x: A class 'x' + :param A_tuple: A class, or a tuple of classes. + :return: A boolean value - 'True' if 'x' is a subclass of 'A_tuple', 'False' otherwise. + Note that if the origin of 'x' is a list or tuple, the function immediately returns 'False'. + """ + if get_origin(x) in (list, tuple): + return False + return issubclass(x, A_tuple) + @staticmethod def _get_values_by_column(docs: Sequence[BaseDoc], col_name: str) -> List[Any]: """Get the value of a column of a document. @@ -776,7 +792,7 @@ def _update_subindex_data( for field_name, type_, _ in self._flatten_schema( cast(Type[BaseDoc], self._schema) ): - if issubclass(type_, AnyDocArray): + if self.custom_issubclass(type_, AnyDocArray): for doc in docs: _list = getattr(doc, field_name) for i, nested_doc in enumerate(_list): @@ -857,11 +873,11 @@ def _flatten_schema( raise ValueError( f'Union type {t_} is not supported. Only Union of subclasses of AbstractTensor or Union[type, None] are supported.' ) - elif issubclass(t_, BaseDoc): + elif BaseDocIndex.custom_issubclass(t_, BaseDoc): names_types_fields.extend( cls._flatten_schema(t_, name_prefix=inner_prefix) ) - elif issubclass(t_, AbstractTensor): + elif BaseDocIndex.custom_issubclass(t_, AbstractTensor): names_types_fields.append( (name_prefix + field_name, AbstractTensor, field_) ) @@ -879,7 +895,7 @@ def _create_column_infos(self, schema: Type[BaseDoc]) -> Dict[str, _ColumnInfo]: column_infos: Dict[str, _ColumnInfo] = dict() for field_name, type_, field_ in self._flatten_schema(schema): # Union types are handle in _flatten_schema - if issubclass(type_, AnyDocArray): + if self.custom_issubclass(type_, AnyDocArray): column_infos[field_name] = _ColumnInfo( docarray_type=type_, db_type=None, config=dict(), n_dim=None ) @@ -921,7 +937,7 @@ def _init_subindex( ): """Initialize subindices if any column is subclass of AnyDocArray.""" for col_name, col in self._column_infos.items(): - if issubclass(col.docarray_type, AnyDocArray): + if BaseDocIndex.custom_issubclass(col.docarray_type, AnyDocArray): sub_db_config = copy.deepcopy(self._db_config) sub_db_config.index_name = f'{self.index_name}__{col_name}' self._subindices[col_name] = self.__class__[col.docarray_type.doc_type]( # type: ignore @@ -1087,7 +1103,7 @@ def _index_subindex(self, column_to_data: Dict[str, Generator[Any, None, None]]) :param column_to_data: A dictionary from column name to a generator """ for col_name, col in self._column_infos.items(): - if issubclass(col.docarray_type, AnyDocArray): + if self.custom_issubclass(col.docarray_type, AnyDocArray): docs = [ doc for doc_list in column_to_data[col_name] for doc in doc_list ] diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index d372aa3134a..5812618ad6d 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -98,7 +98,7 @@ def __init__(self, db_config=None, **kwargs): } self._hnsw_indices = {} for col_name, col in self._column_infos.items(): - if issubclass(col.docarray_type, AnyDocArray): + if BaseDocIndex.custom_issubclass(col.docarray_type, AnyDocArray): continue if not col.config: # non-tensor type; don't create an index @@ -200,7 +200,7 @@ def python_type_to_db_type(self, python_type: Type) -> Any: or None if ``python_type`` is not supported. """ for allowed_type in HNSWLIB_PY_VEC_TYPES: - if issubclass(python_type, allowed_type): + if BaseDocIndex.custom_issubclass(python_type, allowed_type): return np.ndarray return None # all types allowed, but no db type needed @@ -350,7 +350,7 @@ def _del_items(self, doc_ids: Sequence[str]): for field_name, type_, _ in self._flatten_schema( cast(Type[BaseDoc], self._schema) ): - if issubclass(type_, AnyDocArray): + if BaseDocIndex.custom_issubclass(type_, AnyDocArray): for id in doc_ids: doc = self.__getitem__(id) sub_ids = [sub_doc.id for sub_doc in getattr(doc, field_name)] From a2a6d6234c9a43b7b89d2e1f35540da81da1f524 Mon Sep 17 00:00:00 2001 From: maxwelljin2 Date: Thu, 25 May 2023 14:12:00 +0800 Subject: [PATCH 2/7] fix: support non-class data type in document class --- docarray/index/abstract.py | 2 +- docarray/utils/_internal/_typing.py | 4 ++-- tests/index/hnswlib/test_index_get_del.py | 14 +++++++++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 14bf6a115d3..a04fd4d81a9 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -390,7 +390,7 @@ def __delitem__(self, key: Union[str, Sequence[str]]): for field_name, type_, _ in self._flatten_schema( cast(Type[BaseDoc], self._schema) ): - if issubclass(type_, AnyDocArray): + if safe_issubclass(type_, AnyDocArray): for doc_id in key: nested_docs_id = self._subindices[field_name]._filter_by_parent_id( doc_id diff --git a/docarray/utils/_internal/_typing.py b/docarray/utils/_internal/_typing.py index 0a56a24b3cd..2128303bf73 100644 --- a/docarray/utils/_internal/_typing.py +++ b/docarray/utils/_internal/_typing.py @@ -1,6 +1,6 @@ from typing import Any, Optional, get_origin -from typing_inspect import get_args, is_union_type +from typing_inspect import get_args, is_union_type, is_typevar from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -43,6 +43,6 @@ def safe_issubclass(x, a_tuple) -> bool: :return: A boolean value - 'True' if 'x' is a subclass of 'A_tuple', 'False' otherwise. Note that if the origin of 'x' is a list or tuple, the function immediately returns 'False'. """ - if get_origin(x) in (list, tuple): + if (get_origin(x) in (list, tuple, dict, set)) or is_typevar(x): return False return issubclass(x, a_tuple) \ No newline at end of file diff --git a/tests/index/hnswlib/test_index_get_del.py b/tests/index/hnswlib/test_index_get_del.py index c2acb3a5416..29627965b60 100644 --- a/tests/index/hnswlib/test_index_get_del.py +++ b/tests/index/hnswlib/test_index_get_del.py @@ -128,7 +128,7 @@ class TfDoc(BaseDoc): for index in index._hnsw_indices.values(): assert index.get_current_count() == 10 -def test_index_list_str(tmp_path): +def test_index_lst_str(tmp_path): from typing import List class ListDoc(BaseDoc): list_str: List[str] @@ -142,6 +142,18 @@ class ListDoc(BaseDoc): for index in index._hnsw_indices.values(): assert index.get_current_count() == 10 +def test_index_typevar(tmp_path): + from typing import TypeVar + + T = TypeVar("T") + class TypeDoc(BaseDoc): + list_str: T + + index = HnswDocumentIndex[TypeDoc](work_dir=str(tmp_path)) + docs = [TypeDoc(list_str=10) for _ in range(10)] + index.index(docs) + assert index.num_docs() == 10 + def test_index_builtin_docs(tmp_path): # TextDoc class TextSchema(TextDoc): From 1e98c66351bf69a79674c827ec10f983797c438b Mon Sep 17 00:00:00 2001 From: maxwelljin2 Date: Thu, 25 May 2023 15:55:45 +0800 Subject: [PATCH 3/7] fix: added unit tests to issues of non-class input --- docarray/utils/_internal/_typing.py | 2 +- tests/units/util/test_typing.py | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docarray/utils/_internal/_typing.py b/docarray/utils/_internal/_typing.py index 2128303bf73..31c1b34837c 100644 --- a/docarray/utils/_internal/_typing.py +++ b/docarray/utils/_internal/_typing.py @@ -33,7 +33,7 @@ def change_cls_name(cls: type, new_name: str, scope: Optional[dict] = None) -> N cls.__qualname__ = cls.__qualname__[: -len(cls.__name__)] + new_name cls.__name__ = new_name -def safe_issubclass(x, a_tuple) -> bool: +def safe_issubclass(x: type, a_tuple: tuple) -> bool: """ This is a modified version of the built-in 'issubclass' function to support non-class input. Traditional 'issubclass' calls can result in a crash if the input is non-class type (e.g. list/tuple). diff --git a/tests/units/util/test_typing.py b/tests/units/util/test_typing.py index 5446cf3ce04..60b865da3aa 100644 --- a/tests/units/util/test_typing.py +++ b/tests/units/util/test_typing.py @@ -1,10 +1,10 @@ -from typing import Dict, Optional, Union +from typing import Dict, Optional, Union, List, Set, Tuple import pytest from docarray.typing import NdArray, TorchTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal._typing import is_tensor_union, is_type_tensor +from docarray.utils._internal._typing import is_tensor_union, is_type_tensor, safe_issubclass from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() @@ -73,3 +73,17 @@ def test_is_union_type_tensor(type_, is_union_tensor): ) def test_is_union_type_tensor_with_tf(type_, is_union_tensor): assert is_tensor_union(type_) == is_union_tensor + + +@pytest.mark.parametrize( + 'type_, cls, is_subclass', + [ + (List[str], object, False), + (List[List[int]], object, False), + (Set[str], object, False), + (Dict, object, False), + (Tuple[int, int], object, False), + ], +) +def test_safe_issubclass(type_, cls, is_subclass): + assert safe_issubclass(type_, cls) == is_subclass From 1828b91e7c7f2442d8451b3ec70d2ecce4b44e75 Mon Sep 17 00:00:00 2001 From: maxwelljin2 Date: Thu, 25 May 2023 18:25:55 +0800 Subject: [PATCH 4/7] fix: support non-class data input for document class --- docarray/utils/_internal/_typing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docarray/utils/_internal/_typing.py b/docarray/utils/_internal/_typing.py index 31c1b34837c..90e82656031 100644 --- a/docarray/utils/_internal/_typing.py +++ b/docarray/utils/_internal/_typing.py @@ -1,6 +1,6 @@ from typing import Any, Optional, get_origin -from typing_inspect import get_args, is_union_type, is_typevar +from typing_inspect import get_args, is_typevar, is_union_type from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -33,6 +33,7 @@ def change_cls_name(cls: type, new_name: str, scope: Optional[dict] = None) -> N cls.__qualname__ = cls.__qualname__[: -len(cls.__name__)] + new_name cls.__name__ = new_name + def safe_issubclass(x: type, a_tuple: tuple) -> bool: """ This is a modified version of the built-in 'issubclass' function to support non-class input. @@ -45,4 +46,4 @@ def safe_issubclass(x: type, a_tuple: tuple) -> bool: """ if (get_origin(x) in (list, tuple, dict, set)) or is_typevar(x): return False - return issubclass(x, a_tuple) \ No newline at end of file + return issubclass(x, a_tuple) From d7364e4a24ca910d6d2e800b1528d4012459c82f Mon Sep 17 00:00:00 2001 From: maxwelljin2 Date: Thu, 25 May 2023 18:36:35 +0800 Subject: [PATCH 5/7] fix: support non-class data input for document class --- docarray/utils/_internal/_typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/utils/_internal/_typing.py b/docarray/utils/_internal/_typing.py index 90e82656031..13ca424b80f 100644 --- a/docarray/utils/_internal/_typing.py +++ b/docarray/utils/_internal/_typing.py @@ -34,7 +34,7 @@ def change_cls_name(cls: type, new_name: str, scope: Optional[dict] = None) -> N cls.__name__ = new_name -def safe_issubclass(x: type, a_tuple: tuple) -> bool: +def safe_issubclass(x: type, a_tuple: type) -> bool: """ This is a modified version of the built-in 'issubclass' function to support non-class input. Traditional 'issubclass' calls can result in a crash if the input is non-class type (e.g. list/tuple). From 5cd75d2fb5452a468295b53bc93730d4b7d035e5 Mon Sep 17 00:00:00 2001 From: maxwelljin2 Date: Thu, 25 May 2023 18:41:41 +0800 Subject: [PATCH 6/7] fix: support non-class data input for document class Signed-off-by: Ge Jin --- tests/index/hnswlib/test_index_get_del.py | 5 +++++ tests/units/util/test_typing.py | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/index/hnswlib/test_index_get_del.py b/tests/index/hnswlib/test_index_get_del.py index 29627965b60..4ecb60e465a 100644 --- a/tests/index/hnswlib/test_index_get_del.py +++ b/tests/index/hnswlib/test_index_get_del.py @@ -128,8 +128,10 @@ class TfDoc(BaseDoc): for index in index._hnsw_indices.values(): assert index.get_current_count() == 10 + def test_index_lst_str(tmp_path): from typing import List + class ListDoc(BaseDoc): list_str: List[str] @@ -142,10 +144,12 @@ class ListDoc(BaseDoc): for index in index._hnsw_indices.values(): assert index.get_current_count() == 10 + def test_index_typevar(tmp_path): from typing import TypeVar T = TypeVar("T") + class TypeDoc(BaseDoc): list_str: T @@ -154,6 +158,7 @@ class TypeDoc(BaseDoc): index.index(docs) assert index.num_docs() == 10 + def test_index_builtin_docs(tmp_path): # TextDoc class TextSchema(TextDoc): diff --git a/tests/units/util/test_typing.py b/tests/units/util/test_typing.py index 60b865da3aa..0307abb9124 100644 --- a/tests/units/util/test_typing.py +++ b/tests/units/util/test_typing.py @@ -1,10 +1,14 @@ -from typing import Dict, Optional, Union, List, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple, Union import pytest from docarray.typing import NdArray, TorchTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal._typing import is_tensor_union, is_type_tensor, safe_issubclass +from docarray.utils._internal._typing import ( + is_tensor_union, + is_type_tensor, + safe_issubclass, +) from docarray.utils._internal.misc import is_tf_available tf_available = is_tf_available() From 8a13e18ecb323e70dc0a0de7fc54cae69da6088e Mon Sep 17 00:00:00 2001 From: maxwelljin2 Date: Fri, 26 May 2023 11:59:48 +0800 Subject: [PATCH 7/7] fix: support non-class data input for document class Signed-off-by: Ge Jin --- docarray/utils/_internal/_typing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docarray/utils/_internal/_typing.py b/docarray/utils/_internal/_typing.py index 13ca424b80f..a008d562144 100644 --- a/docarray/utils/_internal/_typing.py +++ b/docarray/utils/_internal/_typing.py @@ -1,5 +1,6 @@ -from typing import Any, Optional, get_origin +from typing import Any, Optional +from typing_extensions import get_origin from typing_inspect import get_args, is_typevar, is_union_type from docarray.typing.tensor.abstract_tensor import AbstractTensor