From 1e4f0a7ae3a6e626bb759932a33e173250be8d48 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 27 Apr 2023 17:41:07 +0200 Subject: [PATCH 01/23] fix: allow doclist to have nested optional document Signed-off-by: samsja --- docarray/array/doc_list/doc_list.py | 1 + tests/units/array/test_array.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/docarray/array/doc_list/doc_list.py b/docarray/array/doc_list/doc_list.py index a9deba3bec6..6a33adcf1f4 100644 --- a/docarray/array/doc_list/doc_list.py +++ b/docarray/array/doc_list/doc_list.py @@ -200,6 +200,7 @@ def __class_getitem__(cls, item: Union[Type[BaseDoc], TypeVar, str]): if ( not is_union_type(field_type) + and self.__class__.doc_type.__fields__[field].required and isinstance(field_type, type) and issubclass(field_type, BaseDoc) ): diff --git a/tests/units/array/test_array.py b/tests/units/array/test_array.py index 316baa26d38..765b097bff9 100644 --- a/tests/units/array/test_array.py +++ b/tests/units/array/test_array.py @@ -432,3 +432,23 @@ def test_sort(): assert len(da) == 3 assert da[0].url == 'http://url.com/foo_0.png' assert da[1].url == 'http://url.com/foo_1.png' + + +def test_optional(): + from typing import Optional + + from docarray import BaseDoc, DocList + from docarray.typing import ImageUrl, NdArray + + class Nested(BaseDoc): + tensor: NdArray + + class Image(BaseDoc): + url: ImageUrl + features: Optional[Nested] = None + + docs = DocList[Image]([Image(url='http://url.com/foo.png') for _ in range(10)]) + + assert docs.features == [None for _ in range(10)] + assert isinstance(docs.features, list) + assert not isinstance(docs.features, DocList) From 35b41e0393a8ebfee52451d07c684ba8151e3649 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 2 May 2023 13:46:58 +0200 Subject: [PATCH 02/23] refactor: rename test Signed-off-by: samsja --- tests/units/array/test_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/units/array/test_array.py b/tests/units/array/test_array.py index 765b097bff9..33344bd2aaa 100644 --- a/tests/units/array/test_array.py +++ b/tests/units/array/test_array.py @@ -434,7 +434,7 @@ def test_sort(): assert da[1].url == 'http://url.com/foo_1.png' -def test_optional(): +def test_optional_field(): from typing import Optional from docarray import BaseDoc, DocList From 264bd84b6fc77d32d5b0f59c1f8d9f72dc98a540 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 14:48:56 +0200 Subject: [PATCH 03/23] refactor: allow doc vec column to be None Signed-off-by: samsja --- docarray/array/any_array.py | 2 +- docarray/array/doc_vec/column_storage.py | 34 ++- docarray/array/doc_vec/doc_vec.py | 264 ++++++++++++------ docarray/display/document_array_summary.py | 13 +- tests/units/array/stack/test_array_stacked.py | 48 ++-- 5 files changed, 226 insertions(+), 135 deletions(-) diff --git a/docarray/array/any_array.py b/docarray/array/any_array.py index 4ba2d102cbb..612fba7f42e 100644 --- a/docarray/array/any_array.py +++ b/docarray/array/any_array.py @@ -106,7 +106,7 @@ def __getattr__(self, item: str): def _get_data_column( self: T, field: str, - ) -> Union[MutableSequence, T, 'AbstractTensor']: + ) -> Union[MutableSequence, T, 'AbstractTensor', None]: """Return all values of the fields from all docs this array contains :param field: name of the fields to extract diff --git a/docarray/array/doc_vec/column_storage.py b/docarray/array/doc_vec/column_storage.py index 736b4114b16..164d14427a7 100644 --- a/docarray/array/doc_vec/column_storage.py +++ b/docarray/array/doc_vec/column_storage.py @@ -5,6 +5,7 @@ Dict, Iterable, MutableMapping, + Optional, Type, TypeVar, Union, @@ -37,9 +38,9 @@ class ColumnStorage: def __init__( self, - tensor_columns: Dict[str, AbstractTensor], - doc_columns: Dict[str, 'DocVec'], - docs_vec_columns: Dict[str, ListAdvancedIndexing['DocVec']], + tensor_columns: Dict[str, Optional[AbstractTensor]], + doc_columns: Dict[str, Optional['DocVec']], + docs_vec_columns: Dict[str, Optional[ListAdvancedIndexing['DocVec']]], any_columns: Dict[str, ListAdvancedIndexing], tensor_type: Type[AbstractTensor] = NdArray, ): @@ -63,12 +64,22 @@ def __len__(self) -> int: def __getitem__(self: T, item: IndexIterType) -> T: if isinstance(item, tuple): item = list(item) - tensor_columns = {key: col[item] for key, col in self.tensor_columns.items()} - doc_columns = {key: col[item] for key, col in self.doc_columns.items()} + tensor_columns = { + key: col[item] if col is not None else col + for key, col in self.tensor_columns.items() + } + doc_columns = { + key: col[item] if col is not None else col + for key, col in self.doc_columns.items() + } docs_vec_columns = { - key: col[item] for key, col in self.docs_vec_columns.items() + key: col[item] if col is not None else col + for key, col in self.docs_vec_columns.items() + } + any_columns = { + key: col[item] if col is not None else col + for key, col in self.any_columns.items() } - any_columns = {key: col[item] for key, col in self.any_columns.items()} return self.__class__( tensor_columns, @@ -91,11 +102,18 @@ def __init__(self, index: int, storage: ColumnStorage): def __getitem__(self, name: str) -> Any: if name in self.storage.tensor_columns.keys(): tensor = self.storage.tensor_columns[name] + if tensor is None: + return None if tensor.get_comp_backend().n_dim(tensor) == 1: # to ensure consistensy between numpy and pytorch # we wrap the scalr in a tensor of ndim = 1 # otherwise numpy pass by value whereas torch by reference - return self.storage.tensor_columns[name][self.index : self.index + 1] + col = self.storage.tensor_columns[name] + + if col is not None: + return col[self.index : self.index + 1] + else: + return None return self.storage.columns[name][self.index] diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py index 101fe5b93e3..8f7db6b3cb9 100644 --- a/docarray/array/doc_vec/doc_vec.py +++ b/docarray/array/doc_vec/doc_vec.py @@ -6,6 +6,7 @@ Iterable, List, MutableSequence, + Optional, Sequence, Tuple, Type, @@ -98,9 +99,9 @@ def __init__( ): self.tensor_type = tensor_type - tensor_columns: Dict[str, AbstractTensor] = dict() - doc_columns: Dict[str, 'DocVec'] = dict() - docs_vec_columns: Dict[str, ListAdvancedIndexing['DocVec']] = dict() + tensor_columns: Dict[str, Optional[AbstractTensor]] = dict() + doc_columns: Dict[str, Optional['DocVec']] = dict() + docs_vec_columns: Dict[str, Optional[ListAdvancedIndexing['DocVec']]] = dict() any_columns: Dict[str, ListAdvancedIndexing] = dict() if len(docs) == 0: @@ -116,8 +117,34 @@ def __init__( # from each document and put them in the corresponding column field_type = self.doc_type._get_field_type(field_name) + is_field_required = self.doc_type.__fields__[field_name].required + + first_doc_is_none = getattr(docs[0], field_name) is None + + def _check_none_field(): + + if is_field_required: + if first_doc_is_none: + raise ValueError( + f'Field {field_name} is None for {docs[0]} even though it is required' + ) + + if first_doc_is_none: + for doc in docs: + if getattr(doc, field_name) is not None: + raise ValueError( + f'Field {field_name} is not None for {doc} even though it is None for the first doc' + ) + + def _check_field_not_none(field_name, doc): + if getattr(doc, field_name) is None: + raise ValueError( + f'Field {field_name} is None for {doc} even though it is not None for the first doc' + ) + if is_tensor_union(field_type): field_type = tensor_type + if isinstance(field_type, type): if tf_available and issubclass(field_type, TensorFlowTensor): # tf.Tensor does not allow item assignment, therefore the @@ -125,55 +152,78 @@ def __init__( # of initializing an empty array and assigning values to it # iteratively # does not work here, therefore handle separately. - tf_stack = [] - for i, doc in enumerate(docs): - val = getattr(doc, field_name) - if val is None: - val = TensorFlowTensor( - tensor_type.get_comp_backend().none_value() - ) - tf_stack.append(val.tensor) - stacked: tf.Tensor = tf.stack(tf_stack) - tensor_columns[field_name] = TensorFlowTensor(stacked) + if first_doc_is_none: + _check_none_field() + tensor_columns[field_name] = None + else: + tf_stack = [] + for i, doc in enumerate(docs): + val = getattr(doc, field_name) + _check_field_not_none(field_name, doc) + tf_stack.append(val.tensor) + + stacked: tf.Tensor = tf.stack(tf_stack) + tensor_columns[field_name] = TensorFlowTensor(stacked) elif issubclass(field_type, AbstractTensor): - tensor = getattr(docs[0], field_name) - column_shape = ( - (len(docs), *tensor.shape) - if tensor is not None - else (len(docs),) - ) - tensor_columns[field_name] = field_type._docarray_from_native( - field_type.get_comp_backend().empty( - column_shape, - dtype=tensor.dtype if hasattr(tensor, 'dtype') else None, - device=tensor.device if hasattr(tensor, 'device') else None, + if first_doc_is_none: + _check_none_field() + tensor_columns[field_name] = None + else: + tensor = getattr(docs[0], field_name) + column_shape = ( + (len(docs), *tensor.shape) + if tensor is not None + else (len(docs),) + ) + tensor_columns[field_name] = field_type._docarray_from_native( + field_type.get_comp_backend().empty( + column_shape, + dtype=tensor.dtype + if hasattr(tensor, 'dtype') + else None, + device=tensor.device + if hasattr(tensor, 'device') + else None, + ) ) - ) - - for i, doc in enumerate(docs): - val = getattr(doc, field_name) - if val is None: - val = tensor_type.get_comp_backend().none_value() - cast(AbstractTensor, tensor_columns[field_name])[i] = val + for i, doc in enumerate(docs): + _check_field_not_none(field_name, doc) + val = getattr(doc, field_name) + cast(AbstractTensor, tensor_columns[field_name])[i] = val elif issubclass(field_type, BaseDoc): - doc_columns[field_name] = getattr(docs, field_name).to_doc_vec( - tensor_type=self.tensor_type - ) - - elif issubclass(field_type, AnyDocArray): - docs_list = list() - for doc in docs: - docs_nested = getattr(doc, field_name) - if isinstance(docs_nested, DocList): - docs_nested = docs_nested.to_doc_vec( + if first_doc_is_none: + _check_none_field() + doc_columns[field_name] = None + else: + if is_field_required: + doc_columns[field_name] = getattr( + docs, field_name + ).to_doc_vec(tensor_type=self.tensor_type) + else: + doc_columns[field_name] = DocList.__class_getitem__( + field_type + )(getattr(docs, field_name)).to_doc_vec( tensor_type=self.tensor_type ) - docs_list.append(docs_nested) - docs_vec_columns[field_name] = ListAdvancedIndexing(docs_list) + + elif issubclass(field_type, AnyDocArray): + if first_doc_is_none: + _check_none_field() + doc_columns[field_name] = None + else: + docs_list = list() + for doc in docs: + docs_nested = getattr(doc, field_name) + if isinstance(docs_nested, DocList): + docs_nested = docs_nested.to_doc_vec( + tensor_type=self.tensor_type + ) + docs_list.append(docs_nested) + docs_vec_columns[field_name] = ListAdvancedIndexing(docs_list) else: any_columns[field_name] = ListAdvancedIndexing( getattr(docs, field_name) @@ -227,15 +277,18 @@ def to(self: T, device: str) -> T: :param device: the device to move the data to """ for field, col_tens in self._storage.tensor_columns.items(): - self._storage.tensor_columns[field] = col_tens.get_comp_backend().to_device( - col_tens, device - ) + if col_tens is not None: + self._storage.tensor_columns[ + field + ] = col_tens.get_comp_backend().to_device(col_tens, device) for field, col_doc in self._storage.doc_columns.items(): - self._storage.doc_columns[field] = col_doc.to(device) + if col_doc is not None: + self._storage.doc_columns[field] = col_doc.to(device) for _, col_da in self._storage.docs_vec_columns.items(): - for docs in col_da: - docs.to(device) + if col_da is not None: + for docs in col_da: + docs.to(device) return self @@ -263,7 +316,7 @@ def __getitem__(self: T, item: Union[int, IndexIterType]) -> Union[T_doc, T]: def _get_data_column( self: T, field: str, - ) -> Union[MutableSequence, 'DocVec', AbstractTensor]: + ) -> Union[MutableSequence, 'DocVec', AbstractTensor, None]: """Return one column of the data :param field: name of the fields to extract @@ -353,6 +406,7 @@ def _set_data_column( T, DocList, AbstractTensor, + None, ], ) -> None: """Set all Documents in this DocList using the passed values @@ -360,39 +414,57 @@ def _set_data_column( :param field: name of the fields to set :values: the values to set at the DocList level """ + if values is None: + if field in self._storage.tensor_columns.keys(): + self._storage.tensor_columns[field] = values + elif field in self._storage.doc_columns.keys(): + self._storage.doc_columns[field] = values + elif field in self._storage.docs_vec_columns.keys(): + self._storage.docs_vec_columns[field] = values + elif field in self._storage.any_columns.keys(): + raise ValueError( + f'column {field} cannot be set to None, try to pass ' + f'a list of None instead' + ) + else: + raise ValueError(f'{field} does not exist in {self}') - if len(values) != len(self._storage): - raise ValueError( - f'{values} has not the right length, expected ' - f'{len(self._storage)} , got {len(values)}' - ) - if field in self._storage.tensor_columns.keys(): - validation_class = ( - self._storage.tensor_columns[field].__unparametrizedcls__ - or self._storage.tensor_columns[field].__class__ - ) - # TODO shape check should be handle by the tensor validation + else: + if len(values) != len(self._storage): + raise ValueError( + f'{values} has not the right length, expected ' + f'{len(self._storage)} , got {len(values)}' + ) + if field in self._storage.tensor_columns.keys(): - values = parse_obj_as(validation_class, values) - self._storage.tensor_columns[field] = values + col = self._storage.tensor_columns[field] + if col is not None: + validation_class = col.__unparametrizedcls__ or col.__class__ + else: + validation_class = self.doc_type.__fields__[field].type_ - elif field in self._storage.doc_columns.keys(): - values_ = parse_obj_as( - DocVec.__class_getitem__(self._storage.doc_columns[field].doc_type), - values, - ) - self._storage.doc_columns[field] = values_ + # TODO shape check should be handle by the tensor validation - elif field in self._storage.docs_vec_columns.keys(): - values_ = cast(Sequence[DocList[T_doc]], values) - # TODO here we should actually check if this is correct - self._storage.docs_vec_columns[field] = values_ - elif field in self._storage.any_columns.keys(): - # TODO here we should actually check if this is correct - values_ = cast(Sequence, values) - self._storage.any_columns[field] = values_ - else: - raise KeyError(f'{field} is not a valid field for this DocList') + values = parse_obj_as(validation_class, values) + self._storage.tensor_columns[field] = values + + elif field in self._storage.doc_columns.keys(): + values_ = parse_obj_as( + DocVec.__class_getitem__(self.doc_type._get_field_type(field)), + values, + ) + self._storage.doc_columns[field] = values_ + + elif field in self._storage.docs_vec_columns.keys(): + values_ = cast(Sequence[DocList[T_doc]], values) + # TODO here we should actually check if this is correct + self._storage.docs_vec_columns[field] = values_ + elif field in self._storage.any_columns.keys(): + # TODO here we should actually check if this is correct + values_ = cast(Sequence, values) + self._storage.any_columns[field] = values_ + else: + raise KeyError(f'{field} is not a valid field for this DocList') #################### # Deleting data # @@ -434,7 +506,7 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocVecProto') -> T: return cls.from_columns_storage(storage) def to_protobuf(self) -> 'DocVecProto': - """Convert DocList into a Protobuf message""" + """Convert DocVec into a Protobuf message""" from docarray.proto import ( DocListProto, DocVecProto, @@ -453,13 +525,14 @@ def to_protobuf(self) -> 'DocVecProto': any_columns_proto: Dict[str, ListOfAnyProto] = dict() for field, col_doc in self._storage.doc_columns.items(): - doc_columns_proto[field] = col_doc.to_protobuf() + doc_columns_proto[field] = col_doc.to_protobuf() if col_doc else None for field, col_tens in self._storage.tensor_columns.items(): - tensor_columns_proto[field] = col_tens.to_protobuf() + tensor_columns_proto[field] = col_tens.to_protobuf() if col_tens else None for field, col_da in self._storage.docs_vec_columns.items(): list_proto = ListOfDocArrayProto() - for docs in col_da: - list_proto.data.append(docs.to_protobuf()) + if col_da: + for docs in col_da: + list_proto.data.append(docs.to_protobuf()) da_columns_proto[field] = list_proto for field, col_any in self._storage.any_columns.items(): list_proto = ListOfAnyProto() @@ -480,24 +553,29 @@ def to_doc_list(self: T) -> DocList[T_doc]: Note this destroys the arguments and returns a new DocList """ - unstacked_doc_column: Dict[str, DocList] = dict() - unstacked_da_column: Dict[str, List[DocList]] = dict() - unstacked_tensor_column: Dict[str, List[AbstractTensor]] = dict() + unstacked_doc_column: Dict[str, Optional[DocList]] = dict() + unstacked_da_column: Dict[str, Optional[List[DocList]]] = dict() + unstacked_tensor_column: Dict[str, Optional[List[AbstractTensor]]] = dict() unstacked_any_column = self._storage.any_columns for field, doc_col in self._storage.doc_columns.items(): - unstacked_doc_column[field] = doc_col.to_doc_list() + unstacked_doc_column[field] = doc_col.to_doc_list() if doc_col else None for field, da_col in self._storage.docs_vec_columns.items(): - unstacked_da_column[field] = [docs.to_doc_list() for docs in da_col] + + unstacked_da_column[field] = ( + [docs.to_doc_list() for docs in da_col] if da_col else None + ) for field, tensor_col in list(self._storage.tensor_columns.items()): # list is needed here otherwise we cannot delete the column - unstacked_tensor_column[field] = list() - for tensor in tensor_col: - tensor_copy = tensor.get_comp_backend().copy(tensor) - unstacked_tensor_column[field].append(tensor_copy) + if tensor_col is not None: + tensors = list() + for tensor in tensor_col: + tensor_copy = tensor.get_comp_backend().copy(tensor) + tensors.append(tensor_copy) + unstacked_tensor_column[field] = tensors del self._storage.tensor_columns[field] unstacked_column = ChainMap( # type: ignore diff --git a/docarray/display/document_array_summary.py b/docarray/display/document_array_summary.py index 60d2bc7445d..3454ce2b1f5 100644 --- a/docarray/display/document_array_summary.py +++ b/docarray/display/document_array_summary.py @@ -65,11 +65,12 @@ def _get_stacked_fields(docs: 'DocVec') -> List[str]: # TODO this might for field_name, value_tens in docs._storage.tensor_columns.items(): fields.append(field_name) for field_name, value_doc in docs._storage.doc_columns.items(): - fields.extend( - [ - f'{field_name}.{x}' - for x in DocArraySummary._get_stacked_fields(docs=value_doc) - ] - ) + if value_doc is not None: + fields.extend( + [ + f'{field_name}.{x}' + for x in DocArraySummary._get_stacked_fields(docs=value_doc) + ] + ) return fields diff --git a/tests/units/array/stack/test_array_stacked.py b/tests/units/array/stack/test_array_stacked.py index 67f0bd44ba9..c68713cfcbe 100644 --- a/tests/units/array/stack/test_array_stacked.py +++ b/tests/units/array/stack/test_array_stacked.py @@ -476,35 +476,10 @@ class MyDoc(BaseDoc): assert all(doc.scalar is None for doc in da) assert all(doc.scalar == doc.scalar for doc in da) stacked_da = da.to_doc_vec() - assert type(stacked_da.scalar) == NdArray + assert stacked_da.scalar is None - assert all(type(doc.scalar) == NdArray for doc in stacked_da) # TODO fail here + assert all(doc.scalar is None for doc in stacked_da) # Stacking them turns them into np.nan - assert all(doc.scalar.ndim == 1 for doc in stacked_da) - assert all(doc.scalar != doc.scalar for doc in stacked_da) # Test for nan - - stacked_da.scalar[0] = 3.0 - assert stacked_da[0].scalar == 3.0 - - -def test_torch_nan(): - class MyDoc(BaseDoc): - scalar: Optional[TorchTensor] - - da = DocList[MyDoc]([MyDoc() for _ in range(3)]) - assert all(doc.scalar is None for doc in da) - assert all(doc.scalar == doc.scalar for doc in da) - stacked_da = da.to_doc_vec(tensor_type=TorchTensor) - assert type(stacked_da.scalar) == TorchTensor - - assert all(type(doc.scalar) == TorchTensor for doc in stacked_da) - # Stacking them turns them into torch.nan - assert all(doc.scalar.ndim == 1 for doc in stacked_da) - assert all(doc.scalar != doc.scalar for doc in stacked_da) # Test for nan - - # Make sure they share memory - stacked_da.scalar[0] = 3.0 - assert stacked_da[0].scalar == 3.0 def test_from_storage(): @@ -590,3 +565,22 @@ def test_doc_view_nested(batch_nested_doc): # batch[0].__fields_set__ batch[0].inner = Inner(hello='world') assert batch.inner[0].hello == 'world' + + +def test_optional_field(): + from typing import Optional + + class Nested(BaseDoc): + tensor: NdArray + + class Image(BaseDoc): + url: str + features: Optional[Nested] = None + + docs = DocVec[Image]([Image(url='http://url.com/foo.png') for _ in range(10)]) + + assert docs.features is None + + docs.features = DocList[Nested]([Nested(tensor=np.zeros(10)) for _ in range(10)]) + + assert docs.features.tensor.shape == (10, 10) From 910864f0d7ab61e3439d1c78960bf8f343408395 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 15:05:29 +0200 Subject: [PATCH 04/23] refactor: add better error message when DocVec fail Signed-off-by: samsja --- docarray/array/doc_vec/doc_vec.py | 8 +++++++- tests/units/array/stack/test_array_stacked.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py index 8f7db6b3cb9..8c5d3167cb5 100644 --- a/docarray/array/doc_vec/doc_vec.py +++ b/docarray/array/doc_vec/doc_vec.py @@ -23,7 +23,7 @@ from docarray.array.doc_list.doc_list import DocList from docarray.array.doc_vec.column_storage import ColumnStorage, ColumnStorageView from docarray.array.list_advance_indexing import ListAdvancedIndexing -from docarray.base_doc import BaseDoc +from docarray.base_doc import AnyDoc, BaseDoc from docarray.base_doc.mixins.io import _type_to_protobuf from docarray.typing import NdArray from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -97,6 +97,12 @@ def __init__( docs: Sequence[T_doc], tensor_type: Type['AbstractTensor'] = NdArray, ): + + if not hasattr(self, 'doc_type') or self.doc_type == AnyDoc: + raise TypeError( + f'{self.__class__.__name__} does not precise a doc_type. You probably should do' + f'docs = DocVec[MyDoc](docs) instead of DocVec(docs)' + ) self.tensor_type = tensor_type tensor_columns: Dict[str, Optional[AbstractTensor]] = dict() diff --git a/tests/units/array/stack/test_array_stacked.py b/tests/units/array/stack/test_array_stacked.py index c68713cfcbe..e1c2e138cd0 100644 --- a/tests/units/array/stack/test_array_stacked.py +++ b/tests/units/array/stack/test_array_stacked.py @@ -584,3 +584,9 @@ class Image(BaseDoc): docs.features = DocList[Nested]([Nested(tensor=np.zeros(10)) for _ in range(10)]) assert docs.features.tensor.shape == (10, 10) + + +def test_type_error_no_doc_type(): + + with pytest.raises(TypeError): + DocVec([BaseDoc() for _ in range(10)]) From ba50e171165c6402f0c4afbafdc157e0522a20f9 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 15:59:31 +0200 Subject: [PATCH 05/23] fix: add more test on none mode Signed-off-by: samsja --- docarray/array/doc_vec/column_storage.py | 15 +++- docarray/array/doc_vec/doc_vec.py | 4 +- .../storage/test_array_stack_with_optional.py | 73 +++++++++++++++++++ tests/units/array/stack/test_array_stacked.py | 19 ----- 4 files changed, 89 insertions(+), 22 deletions(-) create mode 100644 tests/units/array/stack/storage/test_array_stack_with_optional.py diff --git a/docarray/array/doc_vec/column_storage.py b/docarray/array/doc_vec/column_storage.py index 164d14427a7..42feac55c95 100644 --- a/docarray/array/doc_vec/column_storage.py +++ b/docarray/array/doc_vec/column_storage.py @@ -115,9 +115,22 @@ def __getitem__(self, name: str) -> Any: else: return None - return self.storage.columns[name][self.index] + col = self.storage.columns[name] + + if col is not None: + return col[self.index] + else: + return None def __setitem__(self, name, value) -> None: + if self.storage.columns[name] is None: + raise ValueError( + f'Cannot set an item to a None column. This mean that ' + f'the DocVec that encapsulate this doc has the field ' + f'{name} set to None. If you want to modify that you need to do it at the' + f'DocVec level. `docs.field = np.zeros(10)`' + ) + self.storage.columns[name][self.index] = value def __delitem__(self, key): diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py index 8c5d3167cb5..226e863d1a7 100644 --- a/docarray/array/doc_vec/doc_vec.py +++ b/docarray/array/doc_vec/doc_vec.py @@ -136,10 +136,10 @@ def _check_none_field(): ) if first_doc_is_none: - for doc in docs: + for i, doc in enumerate(docs): if getattr(doc, field_name) is not None: raise ValueError( - f'Field {field_name} is not None for {doc} even though it is None for the first doc' + f'Field {field_name} is put to None for the first doc. This mean that all of the other docs should have this field set to None as well. This is not the case for {doc} at index {i}' ) def _check_field_not_none(field_name, doc): diff --git a/tests/units/array/stack/storage/test_array_stack_with_optional.py b/tests/units/array/stack/storage/test_array_stack_with_optional.py new file mode 100644 index 00000000000..0175fbbcc2d --- /dev/null +++ b/tests/units/array/stack/storage/test_array_stack_with_optional.py @@ -0,0 +1,73 @@ +from typing import Optional + +import numpy as np +import pytest + +from docarray import BaseDoc, DocList, DocVec +from docarray.typing import NdArray + + +class Nested(BaseDoc): + tensor: NdArray + + +class Image(BaseDoc): + features: Optional[Nested] = None + + +def test_optional_field(): + docs = DocVec[Image]([Image() for _ in range(10)]) + + assert docs.features is None + + docs.features = DocList[Nested]([Nested(tensor=np.zeros(10)) for _ in range(10)]) + + assert docs.features.tensor.shape == (10, 10) + + for doc in docs: + assert doc.features.tensor.shape == (10,) + + +def test_set_none(): + docs = DocVec[Image]( + [Image(features=Nested(tensor=np.zeros(10))) for _ in range(10)] + ) + assert docs.features.tensor.shape == (10, 10) + + docs.features = None + + assert docs.features is None + + for doc in docs: + assert doc.features is None + + +def test_set_doc(): + docs = DocVec[Image]( + [Image(features=Nested(tensor=np.zeros(10))) for _ in range(10)] + ) + assert docs.features.tensor.shape == (10, 10) + + for doc in docs: + doc.features = Nested(tensor=np.ones(10)) + + with pytest.raises(ValueError): + doc.features = None + + +def test_set_doc_none(): + docs = DocVec[Image]([Image() for _ in range(10)]) + + assert docs.features is None + + for doc in docs: + with pytest.raises(ValueError): + doc.features = Nested(tensor=np.ones(10)) + + +def test_no_uniform_none(): + with pytest.raises(ValueError): + DocVec[Image]([Image(), Image(features=Nested(tensor=np.zeros(10)))]) + + with pytest.raises(ValueError): + DocVec[Image]([Image(features=Nested(tensor=np.zeros(10))), Image()]) diff --git a/tests/units/array/stack/test_array_stacked.py b/tests/units/array/stack/test_array_stacked.py index e1c2e138cd0..682ed5e6057 100644 --- a/tests/units/array/stack/test_array_stacked.py +++ b/tests/units/array/stack/test_array_stacked.py @@ -567,25 +567,6 @@ def test_doc_view_nested(batch_nested_doc): assert batch.inner[0].hello == 'world' -def test_optional_field(): - from typing import Optional - - class Nested(BaseDoc): - tensor: NdArray - - class Image(BaseDoc): - url: str - features: Optional[Nested] = None - - docs = DocVec[Image]([Image(url='http://url.com/foo.png') for _ in range(10)]) - - assert docs.features is None - - docs.features = DocList[Nested]([Nested(tensor=np.zeros(10)) for _ in range(10)]) - - assert docs.features.tensor.shape == (10, 10) - - def test_type_error_no_doc_type(): with pytest.raises(TypeError): From 28d40ed28729e2e8477a03a6a1ab648bf7fd2b59 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 16:09:12 +0200 Subject: [PATCH 06/23] docs: update docstring for doc vec Signed-off-by: samsja --- docarray/array/doc_vec/doc_vec.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py index 226e863d1a7..6f3a4fa13e7 100644 --- a/docarray/array/doc_vec/doc_vec.py +++ b/docarray/array/doc_vec/doc_vec.py @@ -66,10 +66,10 @@ class DocVec(AnyDocArray[T_doc]): [`BaseDoc`][docarray.BaseDoc]) will be stored in a column. If the field is a tensor, the data from all Documents will be stored as a single - doc_vec (torch/np/tf) tensor. + (torch/np/tf) tensor. If the tensor field is `AnyTensor` or a Union of tensor types, the - `.tensor_type` will be used to determine the type of the doc_vec column. + `.tensor_type` will be used to determine the type of the column. If the field is another [`BaseDoc`][docarray.BaseDoc] the column will be another `DocVec` that follows the schema of the nested Document. @@ -84,6 +84,15 @@ class DocVec(AnyDocArray[T_doc]): this Document "view" is similar to the behavior of `view = tensor[i]` in numpy/PyTorch. + !!! note + DocVec support optional fields. Nevertheless if a field is optional it needs to + be homogeneous. This means that if the first document has a None value all of the + other documents should have a None value as well. + !!! note + If one field is Optional the column will be stored + * as None if the first doc is as the field as None + * as a normal column otherwise that cannot contain None value + :param docs: a homogeneous sequence of `BaseDoc` :param tensor_type: Tensor Class used to wrap the doc_vec tensors. This is useful if the BaseDoc of this DocVec has some undefined tensor type like From 593578ec2d87f3d5a0a84c4de825b604014c3240 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 16:17:54 +0200 Subject: [PATCH 07/23] docs: add docstring to doc list Signed-off-by: samsja --- docarray/array/doc_list/doc_list.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docarray/array/doc_list/doc_list.py b/docarray/array/doc_list/doc_list.py index f2e12d7cceb..cf8da67a1fc 100644 --- a/docarray/array/doc_list/doc_list.py +++ b/docarray/array/doc_list/doc_list.py @@ -102,6 +102,13 @@ class Image(BaseDoc): del docs[0:5] # remove elements for 0 to 5 from DocList ``` + !!! note + if the DocList is homogeneous and its schema contains nested BaseDoc + (i.e, BaseDoc inside a BaseDoc) but that it is an optional field. Calling + `docs.nested_doc` will return a List of the nested BaseDoc instead of DocList. + This is because the nested field could be None and therefore could not fit into + a DocList. + :param docs: iterable of Document """ From 733bfe6cd981cab634dd093495f3585760100ec0 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 16:50:16 +0200 Subject: [PATCH 08/23] docs: add docs about optional field Signed-off-by: samsja --- docs/user_guide/representing/array.md | 93 +++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index 2b66afea473..916913f3371 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -439,6 +439,99 @@ assert not my_doc.is_view() # False To summarize: you should use `DocVec` when you need to work with contiguous data, and you should use `DocList` when you need to rearrange or extend your data. + +## Dealing with Optional Field + + +Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarray.array.doc_vec.doc_vec.DocVec] support optional fields but they behave differently. + +Let's take an example: + +```python +from docarray.typing import NdArray +import numpy as np + + +class ImageDoc(BaseDoc): + tensor: NdArray + + +class ArticleDoc(BaseDoc): + image: Optional[ImageDoc] + title: str +``` + +In this example `ArticleDoc` has an optional field `image` which is an `ImageDoc`. This means that this field can either +be None or be a `ImageDoc` instance. + +Remember for both DocList and DocVec calling `docs.image` will return a list like object of all the images of the documents. + +For DocList it will just iterate over all the documents and collect the image attribute of each document in a sequence for DocVec it will return the column of the image attribute. + +The question which kind of sequence to you pick when the field is optional, i.e, some of the datapoint could be None ? + +For DocList it will return a list of `Optional[ImageDoc]` instead of a `DocList[ImageDoc]`: + +```python +from docarray import DocList + + +docs = DocList[ArticleDoc]( + [ + ArticleDoc(image=ImageDoc(tensor=np.ones((3, 224, 224))), title="Hello"), + ArticleDoc(image=None, title="World"), + ] +) + +assert docs.image == [ImageDoc(tensor=np.ones((3, 224, 224))), None] +``` + +but for DocVec it is a bit different. Indeed, DocVec store the data for each filed as contiguous column. +This means that DocVec can create a column in only two case: either all the data for a field is None or all the data is not None. + +For the first one the whole column will just be None. In the second The column will be a `DocList[ImageDoc]` + + + +```python +from docarray import DocVec + +docs = DocVec[ArticleDoc]( + [ + ArticleDoc(image=ImageDoc(tensor=np.zeros((3, 224, 224))), title="Hello") + for _ in range(10) + ] +) +assert (docs.image.tensor == np.zeros((3, 224, 224))).all() +``` + +Or it can be None + +```python +docs = DocVec[ArticleDoc]([ArticleDoc(title="Hello") for _ in range(10)]) +assert docs.image is None +``` + +But if you try a mix you will get an error: + +```python +try: + docs = DocVec[ArticleDoc]( + [ + ArticleDoc(image=ImageDoc(tensor=np.ones((3, 224, 224))), title="Hello"), + ArticleDoc(image=None, title="World"), + ] + ) +except ValueError as e: + print(e) +``` + +```bash +None is not a +``` + +--- + See also: From b6c59cf029257794a0b8d2508a753e88475921b3 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 16:57:53 +0200 Subject: [PATCH 09/23] docs: add warning Signed-off-by: samsja --- docs/user_guide/representing/array.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index 916913f3371..8f37dbbe538 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -445,6 +445,10 @@ assert not my_doc.is_view() # False Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarray.array.doc_vec.doc_vec.DocVec] support optional fields but they behave differently. +!!! note + This whole section is specific for nested BaseDoc, DocList and DocVec. + for other data type DocList and DocVec will treat the optional case as a normal case. + Let's take an example: ```python From e07d1f68605bc70a31c365eadfa9c2fc241205c4 Mon Sep 17 00:00:00 2001 From: samsja Date: Wed, 3 May 2023 17:15:17 +0200 Subject: [PATCH 10/23] feat: add integreation test Signed-off-by: samsja --- .../array/test_optional_doc_vec.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/integrations/array/test_optional_doc_vec.py diff --git a/tests/integrations/array/test_optional_doc_vec.py b/tests/integrations/array/test_optional_doc_vec.py new file mode 100644 index 00000000000..727228f47d2 --- /dev/null +++ b/tests/integrations/array/test_optional_doc_vec.py @@ -0,0 +1,31 @@ +from typing import Optional + +import numpy as np + +from docarray import BaseDoc, DocVec +from docarray.typing import ImageUrl, NdArray + + +def test_optional(): + class Features(BaseDoc): + tensor: NdArray[100] + + class Image(BaseDoc): + url: ImageUrl + features: Optional[Features] + + docs = DocVec[Image]([Image(url='http://url.com/foo.png') for _ in range(10)]) + + print(docs.features) # None + + docs.features = [Features(tensor=np.random.random([100])) for _ in range(10)] + print(docs.features) # + assert isinstance(docs.features, DocVec[Features]) + + docs.features.tensor = np.ones((10, 100)) + + assert docs[0].features.tensor.shape == (100,) + + docs.features = None + + assert docs[0].features is None From 43ccfbfc3482cb3392455fe5bff7c7cf4a1ab767 Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Thu, 4 May 2023 09:09:56 +0200 Subject: [PATCH 11/23] fix: apply joahnnes docs suggestion Co-authored-by: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> Signed-off-by: samsja <55492238+samsja@users.noreply.github.com> --- docarray/array/doc_list/doc_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docarray/array/doc_list/doc_list.py b/docarray/array/doc_list/doc_list.py index cf8da67a1fc..07d3d37fedd 100644 --- a/docarray/array/doc_list/doc_list.py +++ b/docarray/array/doc_list/doc_list.py @@ -103,8 +103,8 @@ class Image(BaseDoc): ``` !!! note - if the DocList is homogeneous and its schema contains nested BaseDoc - (i.e, BaseDoc inside a BaseDoc) but that it is an optional field. Calling + If the DocList is homogeneous and its schema contains nested BaseDoc + (i.e, BaseDoc inside a BaseDoc) where the nested Document is `Optional`, calling `docs.nested_doc` will return a List of the nested BaseDoc instead of DocList. This is because the nested field could be None and therefore could not fit into a DocList. From d48275263bf471652a7666c8b9cf9ca5034cdf68 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 4 May 2023 09:11:57 +0200 Subject: [PATCH 12/23] refactor: use none directly Signed-off-by: samsja --- docarray/array/doc_vec/column_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docarray/array/doc_vec/column_storage.py b/docarray/array/doc_vec/column_storage.py index 42feac55c95..de03ebd1402 100644 --- a/docarray/array/doc_vec/column_storage.py +++ b/docarray/array/doc_vec/column_storage.py @@ -65,19 +65,19 @@ def __getitem__(self: T, item: IndexIterType) -> T: if isinstance(item, tuple): item = list(item) tensor_columns = { - key: col[item] if col is not None else col + key: col[item] if col is not None else None for key, col in self.tensor_columns.items() } doc_columns = { - key: col[item] if col is not None else col + key: col[item] if col is not None else None for key, col in self.doc_columns.items() } docs_vec_columns = { - key: col[item] if col is not None else col + key: col[item] if col is not None else None for key, col in self.docs_vec_columns.items() } any_columns = { - key: col[item] if col is not None else col + key: col[item] if col is not None else None for key, col in self.any_columns.items() } From 07bb27b9e1a68cc26977b3ec53f0c0e443cb071d Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Thu, 4 May 2023 09:21:34 +0200 Subject: [PATCH 13/23] feat: apply johannes suggestion Co-authored-by: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> Signed-off-by: samsja <55492238+samsja@users.noreply.github.com> --- docarray/array/doc_vec/column_storage.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docarray/array/doc_vec/column_storage.py b/docarray/array/doc_vec/column_storage.py index de03ebd1402..fad66a5e0f8 100644 --- a/docarray/array/doc_vec/column_storage.py +++ b/docarray/array/doc_vec/column_storage.py @@ -117,10 +117,9 @@ def __getitem__(self, name: str) -> Any: col = self.storage.columns[name] - if col is not None: - return col[self.index] - else: + if col None: return None + return col[self.index] def __setitem__(self, name, value) -> None: if self.storage.columns[name] is None: From 7107ccb8ed27ec797c7e62179b744f4ed92946c7 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 4 May 2023 09:24:00 +0200 Subject: [PATCH 14/23] fix: fix is none Signed-off-by: samsja --- docarray/array/doc_vec/column_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/array/doc_vec/column_storage.py b/docarray/array/doc_vec/column_storage.py index fad66a5e0f8..cd55fb63ea7 100644 --- a/docarray/array/doc_vec/column_storage.py +++ b/docarray/array/doc_vec/column_storage.py @@ -117,7 +117,7 @@ def __getitem__(self, name: str) -> Any: col = self.storage.columns[name] - if col None: + if col is None: return None return col[self.index] From 9947cd24532c26b13f95ba827bce41ccdf639b12 Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Thu, 4 May 2023 09:30:39 +0200 Subject: [PATCH 15/23] feat: apply johannes suggestion Co-authored-by: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> Signed-off-by: samsja <55492238+samsja@users.noreply.github.com> --- docarray/array/doc_vec/doc_vec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py index 6f3a4fa13e7..75d6deba2ac 100644 --- a/docarray/array/doc_vec/doc_vec.py +++ b/docarray/array/doc_vec/doc_vec.py @@ -85,7 +85,7 @@ class DocVec(AnyDocArray[T_doc]): numpy/PyTorch. !!! note - DocVec support optional fields. Nevertheless if a field is optional it needs to + DocVec supports optional fields. Nevertheless if a field is optional it needs to be homogeneous. This means that if the first document has a None value all of the other documents should have a None value as well. !!! note From 1a784502aaa2c8be2e6346d8d45d2923cf1c4964 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 4 May 2023 09:50:17 +0200 Subject: [PATCH 16/23] fix: fix last things Signed-off-by: samsja --- docarray/array/doc_vec/doc_vec.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py index 6f3a4fa13e7..918da73b303 100644 --- a/docarray/array/doc_vec/doc_vec.py +++ b/docarray/array/doc_vec/doc_vec.py @@ -136,7 +136,7 @@ def __init__( first_doc_is_none = getattr(docs[0], field_name) is None - def _check_none_field(): + def _verify_optional_field_of_docs(docs): if is_field_required: if first_doc_is_none: @@ -151,7 +151,7 @@ def _check_none_field(): f'Field {field_name} is put to None for the first doc. This mean that all of the other docs should have this field set to None as well. This is not the case for {doc} at index {i}' ) - def _check_field_not_none(field_name, doc): + def _check_doc_field_not_none(field_name, doc): if getattr(doc, field_name) is None: raise ValueError( f'Field {field_name} is None for {doc} even though it is not None for the first doc' @@ -169,13 +169,13 @@ def _check_field_not_none(field_name, doc): # does not work here, therefore handle separately. if first_doc_is_none: - _check_none_field() + _verify_optional_field_of_docs(docs) tensor_columns[field_name] = None else: tf_stack = [] for i, doc in enumerate(docs): val = getattr(doc, field_name) - _check_field_not_none(field_name, doc) + _check_doc_field_not_none(field_name, doc) tf_stack.append(val.tensor) stacked: tf.Tensor = tf.stack(tf_stack) @@ -183,7 +183,7 @@ def _check_field_not_none(field_name, doc): elif issubclass(field_type, AbstractTensor): if first_doc_is_none: - _check_none_field() + _verify_optional_field_of_docs(docs) tensor_columns[field_name] = None else: tensor = getattr(docs[0], field_name) @@ -205,13 +205,13 @@ def _check_field_not_none(field_name, doc): ) for i, doc in enumerate(docs): - _check_field_not_none(field_name, doc) + _check_doc_field_not_none(field_name, doc) val = getattr(doc, field_name) cast(AbstractTensor, tensor_columns[field_name])[i] = val elif issubclass(field_type, BaseDoc): if first_doc_is_none: - _check_none_field() + _verify_optional_field_of_docs(docs) doc_columns[field_name] = None else: if is_field_required: @@ -227,7 +227,7 @@ def _check_field_not_none(field_name, doc): elif issubclass(field_type, AnyDocArray): if first_doc_is_none: - _check_none_field() + _verify_optional_field_of_docs(docs) doc_columns[field_name] = None else: docs_list = list() @@ -540,9 +540,13 @@ def to_protobuf(self) -> 'DocVecProto': any_columns_proto: Dict[str, ListOfAnyProto] = dict() for field, col_doc in self._storage.doc_columns.items(): - doc_columns_proto[field] = col_doc.to_protobuf() if col_doc else None + doc_columns_proto[field] = ( + col_doc.to_protobuf() if col_doc is not None else None + ) for field, col_tens in self._storage.tensor_columns.items(): - tensor_columns_proto[field] = col_tens.to_protobuf() if col_tens else None + tensor_columns_proto[field] = ( + col_tens.to_protobuf() if col_tens is not None else None + ) for field, col_da in self._storage.docs_vec_columns.items(): list_proto = ListOfDocArrayProto() if col_da: From 49a142d822c0b8e388e9d2ee0b83ef5da6897da7 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 4 May 2023 10:10:25 +0200 Subject: [PATCH 17/23] fix: ad missing check for nested optional doclist Signed-off-by: samsja --- docarray/array/doc_vec/doc_vec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docarray/array/doc_vec/doc_vec.py b/docarray/array/doc_vec/doc_vec.py index 4dce9607e78..e27f6882fe9 100644 --- a/docarray/array/doc_vec/doc_vec.py +++ b/docarray/array/doc_vec/doc_vec.py @@ -233,6 +233,7 @@ def _check_doc_field_not_none(field_name, doc): docs_list = list() for doc in docs: docs_nested = getattr(doc, field_name) + _check_doc_field_not_none(field_name, doc) if isinstance(docs_nested, DocList): docs_nested = docs_nested.to_doc_vec( tensor_type=self.tensor_type From 8ecc1174a9b95f76bc5faa1d7fc7f6ffcd4db778 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 4 May 2023 10:35:12 +0200 Subject: [PATCH 18/23] fix: fix documentation docs Signed-off-by: samsja --- docs/user_guide/representing/array.md | 1 + simple-dl.csv | 3 +++ simple-dl.json | 1 + 3 files changed, 5 insertions(+) create mode 100644 simple-dl.csv create mode 100644 simple-dl.json diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index 8f37dbbe538..1147d0d99fe 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -452,6 +452,7 @@ Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarr Let's take an example: ```python +from typing import Optional from docarray.typing import NdArray import numpy as np diff --git a/simple-dl.csv b/simple-dl.csv new file mode 100644 index 00000000000..c9b99142ccc --- /dev/null +++ b/simple-dl.csv @@ -0,0 +1,3 @@ +id,text +01464df529b0d7a0a591d72f0fae58c1,doc 0 +30756fd3f66b520e87f5ab3cea43aca4,doc 1 diff --git a/simple-dl.json b/simple-dl.json new file mode 100644 index 00000000000..58772d0930d --- /dev/null +++ b/simple-dl.json @@ -0,0 +1 @@ +[{"id":"f38008713ecc8b238c2d62efd5365caa","text":"doc 0"},{"id":"59f3039732bb0c0e7df6e75977a6aeb1","text":"doc 1"}] \ No newline at end of file From 4c8eb1a27e7207cfcd7fa1b4256516a9d73a9f64 Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Thu, 4 May 2023 15:05:33 +0200 Subject: [PATCH 19/23] feat: apply johannes suggestion Co-authored-by: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> Signed-off-by: samsja <55492238+samsja@users.noreply.github.com> --- docs/user_guide/representing/array.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index 1147d0d99fe..38c4e17b200 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -440,7 +440,7 @@ assert not my_doc.is_view() # False or extend your data. -## Dealing with Optional Field +## Dealing with Optional fields Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarray.array.doc_vec.doc_vec.DocVec] support optional fields but they behave differently. @@ -469,9 +469,9 @@ class ArticleDoc(BaseDoc): In this example `ArticleDoc` has an optional field `image` which is an `ImageDoc`. This means that this field can either be None or be a `ImageDoc` instance. -Remember for both DocList and DocVec calling `docs.image` will return a list like object of all the images of the documents. +Remember that for both DocList and DocVec calling `docs.image` will return a list-like object of all the images of the documents. -For DocList it will just iterate over all the documents and collect the image attribute of each document in a sequence for DocVec it will return the column of the image attribute. +For DocList this call will iterate over all the documents and collect the image attribute of each document in a sequence, and for DocVec it will return the already stacked column of the `.image` attribute. The question which kind of sequence to you pick when the field is optional, i.e, some of the datapoint could be None ? @@ -491,10 +491,10 @@ docs = DocList[ArticleDoc]( assert docs.image == [ImageDoc(tensor=np.ones((3, 224, 224))), None] ``` -but for DocVec it is a bit different. Indeed, DocVec store the data for each filed as contiguous column. -This means that DocVec can create a column in only two case: either all the data for a field is None or all the data is not None. +But for DocVec it is a bit different. Indeed, a DocVec stores the data for each filed as contiguous column. +This means that DocVec can create a column in only two cases: either all the data for a field is None or all the data is not None. -For the first one the whole column will just be None. In the second The column will be a `DocList[ImageDoc]` +For the first case the whole column will just be None. In the second case the column will be a `DocList[ImageDoc]` @@ -510,7 +510,7 @@ docs = DocVec[ArticleDoc]( assert (docs.image.tensor == np.zeros((3, 224, 224))).all() ``` -Or it can be None +Or it can be None: ```python docs = DocVec[ArticleDoc]([ArticleDoc(title="Hello") for _ in range(10)]) From 6ef95b66909efca8f0e4487f5df25356535f9fa0 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 4 May 2023 15:48:47 +0200 Subject: [PATCH 20/23] fix: fix docs Signed-off-by: samsja --- docs/user_guide/representing/array.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index 1147d0d99fe..f7b94b6bb0e 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -446,10 +446,27 @@ assert not my_doc.is_view() # False Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarray.array.doc_vec.doc_vec.DocVec] support optional fields but they behave differently. !!! note - This whole section is specific for nested BaseDoc, DocList and DocVec. - for other data type DocList and DocVec will treat the optional case as a normal case. + example of optional field + + ```python + from typing import Optional + from docarray import BaseDoc + + + class MyDoc(BaseDoc): + nested_doc: Optional[BaseDoc] + ``` + +!!! note + Using optional field is slightly different in some case so watch out. But in a nutshell : + + For nested BaseDoc + * DocList will return a list of document if the field is optional and a DocList if the field is not optional + * DocVec will return a DocList if all documents are there, or None if all docs are None. No mix of docs and None allowed! + * DocVec will behave the same with tensor field. -Let's take an example: + +Let's take an example to illustrate the exact behavior ```python from typing import Optional @@ -473,9 +490,7 @@ Remember for both DocList and DocVec calling `docs.image` will return a list lik For DocList it will just iterate over all the documents and collect the image attribute of each document in a sequence for DocVec it will return the column of the image attribute. -The question which kind of sequence to you pick when the field is optional, i.e, some of the datapoint could be None ? - -For DocList it will return a list of `Optional[ImageDoc]` instead of a `DocList[ImageDoc]`: +For DocList it will return a list of `Optional[ImageDoc]` instead of a `DocList[ImageDoc]`, this is because the list can contain None and DocList can't. ```python from docarray import DocList From dcf07d283a9ff2989b56307a7bb5ffd8842580b5 Mon Sep 17 00:00:00 2001 From: samsja Date: Thu, 4 May 2023 15:51:46 +0200 Subject: [PATCH 21/23] fix: fix docs Signed-off-by: samsja --- docs/user_guide/representing/array.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index f7b94b6bb0e..52da34a05c1 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -466,6 +466,8 @@ Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarr * DocVec will behave the same with tensor field. +#### DocList with optional Field + Let's take an example to illustrate the exact behavior ```python @@ -506,7 +508,10 @@ docs = DocList[ArticleDoc]( assert docs.image == [ImageDoc(tensor=np.ones((3, 224, 224))), None] ``` -but for DocVec it is a bit different. Indeed, DocVec store the data for each filed as contiguous column. +#### DocVec with optional Field + + +For DocVec it is a bit different. Indeed, DocVec store the data for each filed as contiguous column. This means that DocVec can create a column in only two case: either all the data for a field is None or all the data is not None. For the first one the whole column will just be None. In the second The column will be a `DocList[ImageDoc]` From cea57ebbc9a3e4ead8fe0eb8f13120c0235fbebe Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Fri, 5 May 2023 14:45:05 +0200 Subject: [PATCH 22/23] feat: apply johannes suggestion Co-authored-by: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> Signed-off-by: samsja <55492238+samsja@users.noreply.github.com> --- docs/user_guide/representing/array.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index 8fa35680985..b478ecd04f6 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -443,10 +443,10 @@ assert not my_doc.is_view() # False ## Dealing with Optional fields -Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarray.array.doc_vec.doc_vec.DocVec] support optional fields but they behave differently. +Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarray.array.doc_vec.doc_vec.DocVec] support nested optional fields but they behave slightly differently. -!!! note - example of optional field +!!! note "Nested optional field" + By a "nested optional field" we mean a document that is contained within another document, and declared as `Optional`: ```python from typing import Optional @@ -457,18 +457,19 @@ Both [`DocList`][docarray.array.doc_list.doc_list.DocList] and [`DocVec`][docarr nested_doc: Optional[BaseDoc] ``` -!!! note - Using optional field is slightly different in some case so watch out. But in a nutshell : +Using nested optional fields differs slightly between DocList and DocVes, so watch out. But in a nutshell: + +When accessing a nested BaseDoc: +* DocList will return a list of documents if the field is optional and a DocList if the field is not optional +* DocVec will return a DocList if all documents are there, or None if all docs are None. No mix of docs and None allowed! +* DocVec will behave the same for a tensor field instead of a BaseDoc - For nested BaseDoc - * DocList will return a list of document if the field is optional and a DocList if the field is not optional - * DocVec will return a DocList if all documents are there, or None if all docs are None. No mix of docs and None allowed! - * DocVec will behave the same with tensor field. +More detailed information is provided in the sections below -#### DocList with optional Field +#### DocList with nested optional Field -Let's take an example to illustrate the exact behavior +Let's take an example to illustrate the exact behavior: ```python from typing import Optional @@ -508,7 +509,7 @@ docs = DocList[ArticleDoc]( assert docs.image == [ImageDoc(tensor=np.ones((3, 224, 224))), None] ``` -#### DocVec with optional Field +#### DocVec with nested optional Field From 38c80cc4afcb7c333fd550d8f337f26080a8bee5 Mon Sep 17 00:00:00 2001 From: samsja Date: Fri, 5 May 2023 14:54:13 +0200 Subject: [PATCH 23/23] docs: fix mistake Signed-off-by: samsja --- docs/user_guide/representing/array.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_guide/representing/array.md b/docs/user_guide/representing/array.md index b478ecd04f6..7a49c68ea2d 100644 --- a/docs/user_guide/representing/array.md +++ b/docs/user_guide/representing/array.md @@ -461,10 +461,10 @@ Using nested optional fields differs slightly between DocList and DocVes, so wat When accessing a nested BaseDoc: * DocList will return a list of documents if the field is optional and a DocList if the field is not optional -* DocVec will return a DocList if all documents are there, or None if all docs are None. No mix of docs and None allowed! +* DocVec will return a DocVec if all documents are there, or None if all docs are None. No mix of docs and None allowed! * DocVec will behave the same for a tensor field instead of a BaseDoc -More detailed information is provided in the sections below + #### DocList with nested optional Field