From d1f651bad9b6f57b9adeb8914599474c4c31e355 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Feb 2023 11:52:07 +0100 Subject: [PATCH 01/12] test: add test for find on nested attribute Signed-off-by: anna-charlotte --- tests/units/util/test_find.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/units/util/test_find.py b/tests/units/util/test_find.py index 414ec074186..715ec45d2be 100644 --- a/tests/units/util/test_find.py +++ b/tests/units/util/test_find.py @@ -292,6 +292,33 @@ class MyDoc(BaseDocument): assert (torch.stack(sorted(scores, reverse=True)) == scores).all() +def test_find_nested(): + class InnerDoc(BaseDocument): + title: str + embedding: NdArray + + class MyDoc(BaseDocument): + inner: InnerDoc + + query = MyDoc(inner=InnerDoc(title='query', embedding=np.random.randn((10)))) + index = DocumentArray[MyDoc]( + [ + MyDoc(inner=InnerDoc(title=f'doc {i}', embedding=np.random.randn((10)))) + for i in range(10) + ] + ) + + top_k, scores = find( + index, + query, + embedding_field='inner__embedding', + limit=7, + ) + assert len(top_k) == 7 + assert len(scores) == 7 + assert (torch.stack(sorted(scores, reverse=True)) == scores).all() + + def test_find_nested_union_optional(): class MyDoc(BaseDocument): embedding: Union[Optional[TorchTensor], Optional[NdArray]] From e7c417894eeb3b96a3a7ee4669aec96ce06eb92c Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Feb 2023 16:42:24 +0100 Subject: [PATCH 02/12] test: add test for nested attr in stacked da Signed-off-by: anna-charlotte --- tests/units/util/test_find.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/units/util/test_find.py b/tests/units/util/test_find.py index 715ec45d2be..efa4a9f9438 100644 --- a/tests/units/util/test_find.py +++ b/tests/units/util/test_find.py @@ -292,21 +292,24 @@ class MyDoc(BaseDocument): assert (torch.stack(sorted(scores, reverse=True)) == scores).all() -def test_find_nested(): +@pytest.mark.parametrize('stack', [False, True]) +def test_find_nested(stack): class InnerDoc(BaseDocument): title: str - embedding: NdArray + embedding: TorchTensor class MyDoc(BaseDocument): inner: InnerDoc - query = MyDoc(inner=InnerDoc(title='query', embedding=np.random.randn((10)))) + query = MyDoc(inner=InnerDoc(title='query', embedding=torch.rand(2))) index = DocumentArray[MyDoc]( [ - MyDoc(inner=InnerDoc(title=f'doc {i}', embedding=np.random.randn((10)))) + MyDoc(inner=InnerDoc(title=f'doc {i}', embedding=torch.rand(2))) for i in range(10) ] ) + if stack: + index = index.stack() top_k, scores = find( index, From 5ba18576a9d445d23c12d71227f8f59a798cece6 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Feb 2023 16:50:33 +0100 Subject: [PATCH 03/12] fix: nested access Signed-off-by: anna-charlotte --- docarray/helper.py | 18 ++++++++++++++++++ docarray/utils/find.py | 29 ++++++++++++++++------------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/docarray/helper.py b/docarray/helper.py index cd59e331e6e..b1ba0dc56b8 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -121,3 +121,21 @@ def _update_nested_dicts( to_update[k] = v else: _update_nested_dicts(to_update[k], update_with[k]) + + +def _get_field_type(doc_type, access_path): + from docarray import BaseDocument + + field, _, remaining = access_path.partition('__') + if len(remaining) == 0: + return doc_type._get_field_type(field) + else: + valid_field = field in doc_type.__fields__.keys() + if not valid_field: + return None + else: + d = doc_type._get_field_type(field) + if not issubclass(d, BaseDocument): + return None + else: + return _get_field_type(d, remaining) diff --git a/docarray/utils/find.py b/docarray/utils/find.py index 60025ca9c19..8dada575b2e 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -6,6 +6,7 @@ from docarray.array.array.array import DocumentArray from docarray.array.stacked.array_stacked import DocumentArrayStacked from docarray.base_document import BaseDocument +from docarray.helper import _get_field_type from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -192,8 +193,8 @@ class MyDocument(BaseDocument): comp_backend = embedding_type.get_comp_backend() # extract embeddings from query and index - index_embeddings = _extraxt_embeddings(index, embedding_field, embedding_type) - query_embeddings = _extraxt_embeddings(query, embedding_field, embedding_type) + index_embeddings = _extract_embeddings(index, embedding_field, embedding_type) + query_embeddings = _extract_embeddings(query, embedding_field, embedding_type) # compute distances and return top results metric_fn = getattr(comp_backend.Metrics, metric) @@ -225,7 +226,7 @@ def _extract_embedding_single( :return: the embeddings """ if isinstance(data, BaseDocument): - emb = getattr(data, embedding_field) + emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] else: # treat data as tensor emb = data if len(emb.shape) == 1: @@ -235,7 +236,7 @@ def _extract_embedding_single( return emb -def _extraxt_embeddings( +def _extract_embeddings( data: Union[AnyDocumentArray, BaseDocument, AnyTensor], embedding_field: str, embedding_type: Type, @@ -247,14 +248,13 @@ def _extraxt_embeddings( :param embedding_type: type of the embedding: torch.Tensor, numpy.ndarray etc. :return: the embeddings """ - if isinstance(data, DocumentArray): - emb = getattr(data, embedding_field) + emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)] emb = embedding_type._docarray_stack(emb) elif isinstance(data, DocumentArrayStacked): - emb = getattr(data, embedding_field) + emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] elif isinstance(data, BaseDocument): - emb = getattr(data, embedding_field) + emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] else: # treat data as tensor emb = data @@ -265,22 +265,25 @@ def _extraxt_embeddings( return emb -def _da_attr_type(da: AnyDocumentArray, attr: str) -> Type[AnyTensor]: +def _da_attr_type(da: AnyDocumentArray, access_path: str) -> Type[AnyTensor]: """Get the type of the attribute according to the Document type (schema) of the DocumentArray. :param da: the DocumentArray - :param attr: the attribute name + :param access_path: the attribute name :return: the type of the attribute """ - field_type = da.document_type._get_field_type(attr) + field_type = _get_field_type(da.document_type, access_path) + if is_union_type(field_type): # determine type based on the fist element - field_type = type(getattr(da[0], attr)) + field_type = type( + [x for x in AnyDocumentArray._traverse(da[0], access_path)][0] + ) if not issubclass(field_type, AbstractTensor): raise ValueError( - f'attribute {attr} is not a tensor-like type, ' + f'attribute {access_path} is not a tensor-like type, ' f'but {field_type.__class__.__name__}' ) From 7c9a6172a004a7019f257b161212a5682fb2c323 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Feb 2023 17:04:21 +0100 Subject: [PATCH 04/12] fix: mypy Signed-off-by: anna-charlotte --- docarray/utils/find.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docarray/utils/find.py b/docarray/utils/find.py index 8dada575b2e..5136da0e11e 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -248,20 +248,19 @@ def _extract_embeddings( :param embedding_type: type of the embedding: torch.Tensor, numpy.ndarray etc. :return: the embeddings """ + emb: AnyTensor if isinstance(data, DocumentArray): - emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)] - emb = embedding_type._docarray_stack(emb) + emb_list = [x for x in AnyDocumentArray._traverse(data, embedding_field)] + emb = embedding_type._docarray_stack(emb_list) elif isinstance(data, DocumentArrayStacked): emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] elif isinstance(data, BaseDocument): emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] else: # treat data as tensor - emb = data + emb = cast(AnyTensor, data) if len(emb.shape) == 1: - # all currently supported frameworks provide `.reshape()`. Onc this is not true - # anymore, we need to add a `.reshape()` method to the computational backend - emb = emb.reshape(1, -1) + emb = emb.get_comp_backend().reshape(array=emb, shape=(1, -1)) return emb From 5c7e862fe2cfe713837dbd86baf908368a7efabf Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 09:29:55 +0100 Subject: [PATCH 05/12] fix: field type by access path for da Signed-off-by: anna-charlotte --- docarray/helper.py | 36 +++++---------------------------- docarray/utils/find.py | 41 +++++++++++++++++++++++++++++--------- tests/units/test_helper.py | 10 ++++++++-- 3 files changed, 45 insertions(+), 42 deletions(-) diff --git a/docarray/helper.py b/docarray/helper.py index b1ba0dc56b8..57f8ce41522 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -8,21 +8,13 @@ def _is_access_path_valid(doc_type: Type['BaseDocument'], access_path: str) -> b """ Check if a given access path ("__"-separated) is a valid path for a given Document class. """ - from docarray import BaseDocument + from docarray.utils.find import _get_field_type_by_access_path - field, _, remaining = access_path.partition('__') - if len(remaining) == 0: - return access_path in doc_type.__fields__.keys() + field_type = _get_field_type_by_access_path(doc_type, access_path) + if field_type is None: + return False else: - valid_field = field in doc_type.__fields__.keys() - if not valid_field: - return False - else: - d = doc_type._get_field_type(field) - if not issubclass(d, BaseDocument): - return False - else: - return _is_access_path_valid(d, remaining) + return True def _all_access_paths_valid( @@ -121,21 +113,3 @@ def _update_nested_dicts( to_update[k] = v else: _update_nested_dicts(to_update[k], update_with[k]) - - -def _get_field_type(doc_type, access_path): - from docarray import BaseDocument - - field, _, remaining = access_path.partition('__') - if len(remaining) == 0: - return doc_type._get_field_type(field) - else: - valid_field = field in doc_type.__fields__.keys() - if not valid_field: - return None - else: - d = doc_type._get_field_type(field) - if not issubclass(d, BaseDocument): - return None - else: - return _get_field_type(d, remaining) diff --git a/docarray/utils/find.py b/docarray/utils/find.py index 5136da0e11e..9a7ebc6d7ca 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -1,4 +1,4 @@ -from typing import List, NamedTuple, Optional, Type, Union, cast +from typing import Any, List, NamedTuple, Optional, Type, Union, cast from typing_inspect import is_union_type @@ -6,7 +6,6 @@ from docarray.array.array.array import DocumentArray from docarray.array.stacked.array_stacked import DocumentArrayStacked from docarray.base_document import BaseDocument -from docarray.helper import _get_field_type from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -253,9 +252,11 @@ def _extract_embeddings( emb_list = [x for x in AnyDocumentArray._traverse(data, embedding_field)] emb = embedding_type._docarray_stack(emb_list) elif isinstance(data, DocumentArrayStacked): - emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] + emb = next(AnyDocumentArray._traverse(data, embedding_field)) + # emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] elif isinstance(data, BaseDocument): - emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] + emb = next(AnyDocumentArray._traverse(data, embedding_field)) + # emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] else: # treat data as tensor emb = cast(AnyTensor, data) @@ -269,16 +270,14 @@ def _da_attr_type(da: AnyDocumentArray, access_path: str) -> Type[AnyTensor]: (schema) of the DocumentArray. :param da: the DocumentArray - :param access_path: the attribute name + :param access_path: the "__"-separated access path :return: the type of the attribute """ - field_type = _get_field_type(da.document_type, access_path) + field_type = _get_field_type_by_access_path(da.document_type, access_path) if is_union_type(field_type): # determine type based on the fist element - field_type = type( - [x for x in AnyDocumentArray._traverse(da[0], access_path)][0] - ) + field_type = type(next(AnyDocumentArray._traverse(da[0], access_path))) if not issubclass(field_type, AbstractTensor): raise ValueError( @@ -287,3 +286,27 @@ def _da_attr_type(da: AnyDocumentArray, access_path: str) -> Type[AnyTensor]: ) return cast(Type[AnyTensor], field_type) + + +def _get_field_type_by_access_path( + doc_type: Type[BaseDocument], access_path: str +) -> Any: + """ + Get field type by "__"-separated access path. + """ + from docarray import BaseDocument + + field, _, remaining = access_path.partition('__') + field_valid = field in doc_type.__fields__.keys() + + if field_valid: + if len(remaining) == 0: + return doc_type._get_field_type(field) + else: + d = doc_type._get_field_type(field) + if issubclass(d, DocumentArray): + return _get_field_type_by_access_path(d.document_type, remaining) + elif issubclass(d, BaseDocument): + return _get_field_type_by_access_path(d, remaining) + else: + return None diff --git a/tests/units/test_helper.py b/tests/units/test_helper.py index 03098d84282..361f136e817 100644 --- a/tests/units/test_helper.py +++ b/tests/units/test_helper.py @@ -2,7 +2,7 @@ import pytest -from docarray import BaseDocument +from docarray import BaseDocument, DocumentArray from docarray.documents import Image from docarray.helper import ( _access_path_dict_to_nested_dict, @@ -25,8 +25,13 @@ class Middle(BaseDocument): class Outer(BaseDocument): img: Optional[Image] middle: Optional[Middle] + da: DocumentArray[Inner] - doc = Outer(img=Image(), middle=Middle(img=Image(), inner=Inner(img=Image()))) + doc = Outer( + img=Image(), + middle=Middle(img=Image(), inner=Inner(img=Image())), + da=DocumentArray[Inner]([Inner(img=Image(url='test.png'))]), + ) return doc @@ -35,6 +40,7 @@ def test_is_access_path_valid(nested_doc): assert _is_access_path_valid(nested_doc.__class__, 'middle__img') assert _is_access_path_valid(nested_doc.__class__, 'middle__inner__img') assert _is_access_path_valid(nested_doc.__class__, 'middle') + assert _is_access_path_valid(nested_doc.__class__, 'da__img__url') def test_is_access_path_not_valid(nested_doc): From e8a00032ec89d55e78b6d614800125c70b7cff6e Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 09:34:43 +0100 Subject: [PATCH 06/12] fix: clean up Signed-off-by: anna-charlotte --- docarray/utils/find.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docarray/utils/find.py b/docarray/utils/find.py index 9a7ebc6d7ca..09686a6d759 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -225,7 +225,7 @@ def _extract_embedding_single( :return: the embeddings """ if isinstance(data, BaseDocument): - emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] + emb = next(AnyDocumentArray._traverse(data, embedding_field)) else: # treat data as tensor emb = data if len(emb.shape) == 1: @@ -253,10 +253,8 @@ def _extract_embeddings( emb = embedding_type._docarray_stack(emb_list) elif isinstance(data, DocumentArrayStacked): emb = next(AnyDocumentArray._traverse(data, embedding_field)) - # emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] elif isinstance(data, BaseDocument): emb = next(AnyDocumentArray._traverse(data, embedding_field)) - # emb = [x for x in AnyDocumentArray._traverse(data, embedding_field)][0] else: # treat data as tensor emb = cast(AnyTensor, data) @@ -293,6 +291,9 @@ def _get_field_type_by_access_path( ) -> Any: """ Get field type by "__"-separated access path. + :param doc_type: type of document + :param access_path: "__"-separated access path + :return: field type of accessed attribute. If access path is invalid, return None. """ from docarray import BaseDocument From 8ac482755fa0a1cdf531174710d44f67aebc2ba4 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 09:42:45 +0100 Subject: [PATCH 07/12] fix: move get field type by access path Signed-off-by: anna-charlotte --- docarray/helper.py | 30 ++++++++++++++++++++++++++++-- docarray/utils/find.py | 30 ++---------------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/docarray/helper.py b/docarray/helper.py index 57f8ce41522..ca61f1c8b8e 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -1,14 +1,13 @@ from typing import TYPE_CHECKING, Any, Dict, List, Type if TYPE_CHECKING: - from docarray import BaseDocument + from docarray import BaseDocument, DocumentArray def _is_access_path_valid(doc_type: Type['BaseDocument'], access_path: str) -> bool: """ Check if a given access path ("__"-separated) is a valid path for a given Document class. """ - from docarray.utils.find import _get_field_type_by_access_path field_type = _get_field_type_by_access_path(doc_type, access_path) if field_type is None: @@ -113,3 +112,30 @@ def _update_nested_dicts( to_update[k] = v else: _update_nested_dicts(to_update[k], update_with[k]) + + +def _get_field_type_by_access_path( + doc_type: Type[BaseDocument], access_path: str +) -> Any: + """ + Get field type by "__"-separated access path. + :param doc_type: type of document + :param access_path: "__"-separated access path + :return: field type of accessed attribute. If access path is invalid, return None. + """ + from docarray import BaseDocument + + field, _, remaining = access_path.partition('__') + field_valid = field in doc_type.__fields__.keys() + + if field_valid: + if len(remaining) == 0: + return doc_type._get_field_type(field) + else: + d = doc_type._get_field_type(field) + if issubclass(d, DocumentArray): + return _get_field_type_by_access_path(d.document_type, remaining) + elif issubclass(d, BaseDocument): + return _get_field_type_by_access_path(d, remaining) + else: + return None diff --git a/docarray/utils/find.py b/docarray/utils/find.py index 09686a6d759..a579c567aeb 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -1,4 +1,4 @@ -from typing import Any, List, NamedTuple, Optional, Type, Union, cast +from typing import List, NamedTuple, Optional, Type, Union, cast from typing_inspect import is_union_type @@ -6,6 +6,7 @@ from docarray.array.array.array import DocumentArray from docarray.array.stacked.array_stacked import DocumentArrayStacked from docarray.base_document import BaseDocument +from docarray.helper import _get_field_type_by_access_path from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -284,30 +285,3 @@ def _da_attr_type(da: AnyDocumentArray, access_path: str) -> Type[AnyTensor]: ) return cast(Type[AnyTensor], field_type) - - -def _get_field_type_by_access_path( - doc_type: Type[BaseDocument], access_path: str -) -> Any: - """ - Get field type by "__"-separated access path. - :param doc_type: type of document - :param access_path: "__"-separated access path - :return: field type of accessed attribute. If access path is invalid, return None. - """ - from docarray import BaseDocument - - field, _, remaining = access_path.partition('__') - field_valid = field in doc_type.__fields__.keys() - - if field_valid: - if len(remaining) == 0: - return doc_type._get_field_type(field) - else: - d = doc_type._get_field_type(field) - if issubclass(d, DocumentArray): - return _get_field_type_by_access_path(d.document_type, remaining) - elif issubclass(d, BaseDocument): - return _get_field_type_by_access_path(d, remaining) - else: - return None From 2cc5a21b47a09e122bb690edd5b25b264f1fadeb Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 09:44:45 +0100 Subject: [PATCH 08/12] fix: imports Signed-off-by: anna-charlotte --- docarray/helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/helper.py b/docarray/helper.py index ca61f1c8b8e..2b9f46240ad 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -115,7 +115,7 @@ def _update_nested_dicts( def _get_field_type_by_access_path( - doc_type: Type[BaseDocument], access_path: str + doc_type: Type['BaseDocument'], access_path: str ) -> Any: """ Get field type by "__"-separated access path. From 234cb360f912e2a2d6b384cc85af3528cb94f3f5 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 09:51:46 +0100 Subject: [PATCH 09/12] fix: apply suggestion and fix import Signed-off-by: anna-charlotte --- docarray/helper.py | 4 ++-- docarray/utils/find.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docarray/helper.py b/docarray/helper.py index 2b9f46240ad..f0c48bd388c 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Type if TYPE_CHECKING: - from docarray import BaseDocument, DocumentArray + from docarray import BaseDocument def _is_access_path_valid(doc_type: Type['BaseDocument'], access_path: str) -> bool: @@ -123,7 +123,7 @@ def _get_field_type_by_access_path( :param access_path: "__"-separated access path :return: field type of accessed attribute. If access path is invalid, return None. """ - from docarray import BaseDocument + from docarray import BaseDocument, DocumentArray field, _, remaining = access_path.partition('__') field_valid = field in doc_type.__fields__.keys() diff --git a/docarray/utils/find.py b/docarray/utils/find.py index a579c567aeb..7611b1a4675 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -250,7 +250,7 @@ def _extract_embeddings( """ emb: AnyTensor if isinstance(data, DocumentArray): - emb_list = [x for x in AnyDocumentArray._traverse(data, embedding_field)] + emb_list = list(AnyDocumentArray._traverse(data, embedding_field)) emb = embedding_type._docarray_stack(emb_list) elif isinstance(data, DocumentArrayStacked): emb = next(AnyDocumentArray._traverse(data, embedding_field)) From 95a0bd0dfc8fc4b678facefca924f12e9581b903 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 10:28:39 +0100 Subject: [PATCH 10/12] fix: apply suggestion from code review Signed-off-by: anna-charlotte --- docarray/utils/find.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docarray/utils/find.py b/docarray/utils/find.py index 7611b1a4675..b67fe751ae0 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -252,9 +252,7 @@ def _extract_embeddings( if isinstance(data, DocumentArray): emb_list = list(AnyDocumentArray._traverse(data, embedding_field)) emb = embedding_type._docarray_stack(emb_list) - elif isinstance(data, DocumentArrayStacked): - emb = next(AnyDocumentArray._traverse(data, embedding_field)) - elif isinstance(data, BaseDocument): + elif isinstance(data, (DocumentArrayStacked, BaseDocument)): emb = next(AnyDocumentArray._traverse(data, embedding_field)) else: # treat data as tensor emb = cast(AnyTensor, data) From c5206303af524b57a7db433e14482064892a0641 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 10:29:52 +0100 Subject: [PATCH 11/12] fix: apply suggestion from code review Signed-off-by: anna-charlotte --- docarray/helper.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docarray/helper.py b/docarray/helper.py index f0c48bd388c..63f32db0a63 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -10,10 +10,7 @@ def _is_access_path_valid(doc_type: Type['BaseDocument'], access_path: str) -> b """ field_type = _get_field_type_by_access_path(doc_type, access_path) - if field_type is None: - return False - else: - return True + return field_type is not None def _all_access_paths_valid( From 8400ed4037127fc46ccb1da3cd23cc23a9cf7309 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Feb 2023 14:44:47 +0100 Subject: [PATCH 12/12] fix: apply suggestions Signed-off-by: anna-charlotte --- docarray/helper.py | 6 ++++-- docarray/utils/find.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docarray/helper.py b/docarray/helper.py index 63f32db0a63..49ea70cd9b7 100644 --- a/docarray/helper.py +++ b/docarray/helper.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Type +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type if TYPE_CHECKING: from docarray import BaseDocument @@ -113,7 +113,7 @@ def _update_nested_dicts( def _get_field_type_by_access_path( doc_type: Type['BaseDocument'], access_path: str -) -> Any: +) -> Optional[Type]: """ Get field type by "__"-separated access path. :param doc_type: type of document @@ -134,5 +134,7 @@ def _get_field_type_by_access_path( return _get_field_type_by_access_path(d.document_type, remaining) elif issubclass(d, BaseDocument): return _get_field_type_by_access_path(d, remaining) + else: + return None else: return None diff --git a/docarray/utils/find.py b/docarray/utils/find.py index b67fe751ae0..f471255ad7d 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -270,7 +270,11 @@ def _da_attr_type(da: AnyDocumentArray, access_path: str) -> Type[AnyTensor]: :param access_path: the "__"-separated access path :return: the type of the attribute """ - field_type = _get_field_type_by_access_path(da.document_type, access_path) + field_type: Optional[Type] = _get_field_type_by_access_path( + da.document_type, access_path + ) + if field_type is None: + raise ValueError(f"Access path is not valid: {access_path}") if is_union_type(field_type): # determine type based on the fist element