From 03b89dc343acef828b733dbdcd754be5c4ffc384 Mon Sep 17 00:00:00 2001 From: Sami Jaghouar Date: Mon, 16 Jan 2023 13:56:46 +0100 Subject: [PATCH 1/9] feat: add legacy document Signed-off-by: Sami Jaghouar --- docarray/documents/legacy/__init__.py | 6 ++++++ docarray/documents/legacy/legacy_document.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 docarray/documents/legacy/__init__.py create mode 100644 docarray/documents/legacy/legacy_document.py diff --git a/docarray/documents/legacy/__init__.py b/docarray/documents/legacy/__init__.py new file mode 100644 index 00000000000..65ff29d224b --- /dev/null +++ b/docarray/documents/legacy/__init__.py @@ -0,0 +1,6 @@ +from docarray import DocumentArray as BaseDocumentArray +from docarray.documents.legacy.legacy_document import Document + +DocumentArray = BaseDocumentArray[Document] + +__all__ = ['Document', 'DocumentArray'] diff --git a/docarray/documents/legacy/legacy_document.py b/docarray/documents/legacy/legacy_document.py new file mode 100644 index 00000000000..643d525dac5 --- /dev/null +++ b/docarray/documents/legacy/legacy_document.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional + +from docarray import BaseDocument, DocumentArray +from docarray.typing import AnyEmbedding, AnyTensor + + +class Document(BaseDocument): + + tensor: Optional[AnyTensor] + chunks: Optional[DocumentArray[Document]] + matches: Optional[DocumentArray[Document]] + blob: Optional[Any] + text: Optional[str] + url: Optional[str] + embedding: Optional[AnyEmbedding] + tags: Dict[str, Any] = dict() + scores: Optional[Dict[str, Any]] From 4836ba6daa3bb6adad635850a6c464ab7fdd69e7 Mon Sep 17 00:00:00 2001 From: Sami Jaghouar Date: Mon, 16 Jan 2023 15:35:06 +0100 Subject: [PATCH 2/9] feat: add legagy document Signed-off-by: Sami Jaghouar --- docarray/documents/legacy/legacy_document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/documents/legacy/legacy_document.py b/docarray/documents/legacy/legacy_document.py index 643d525dac5..7f878c1edc3 100644 --- a/docarray/documents/legacy/legacy_document.py +++ b/docarray/documents/legacy/legacy_document.py @@ -15,5 +15,5 @@ class Document(BaseDocument): text: Optional[str] url: Optional[str] embedding: Optional[AnyEmbedding] - tags: Dict[str, Any] = dict() + # tags: Dict[str, Any] = dict() scores: Optional[Dict[str, Any]] From 713ea534258d799c7184be3602510813bd8fb104 Mon Sep 17 00:00:00 2001 From: Sami Jaghouar Date: Fri, 20 Jan 2023 17:40:58 +0100 Subject: [PATCH 3/9] fix: latest commit did not solve ci maybe this one Signed-off-by: Sami Jaghouar --- tests/integrations/externals/test_fastapi.py | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/integrations/externals/test_fastapi.py b/tests/integrations/externals/test_fastapi.py index f74f8519a9c..c15e0632c91 100644 --- a/tests/integrations/externals/test_fastapi.py +++ b/tests/integrations/externals/test_fastapi.py @@ -6,7 +6,7 @@ from docarray import BaseDocument from docarray.base_document import DocumentResponse from docarray.documents import Image, Text -from docarray.typing import NdArray +from docarray.typing import AnyTensor, NdArray @pytest.mark.asyncio @@ -107,3 +107,27 @@ async def create_item(doc: InputDoc) -> OutputDoc: assert isinstance(doc, OutputDoc) assert doc.embedding_clip.shape == (100, 1) assert doc.embedding_bert.shape == (100, 1) + + +@pytest.mark.asyncio +async def test_image(): + class InputDoc(BaseDocument): + img: Image + + class OutputDoc(BaseDocument): + embedding: AnyTensor + + input_doc = InputDoc(img=Image(tensor=np.zeros((3, 224, 224)))) + + app = FastAPI() + + @app.post("/embed/", response_model=OutputDoc, response_class=DocumentResponse) + async def embed(doc: InputDoc) -> OutputDoc: + doc = OutputDoc(embedding=np.zeros((100, 1))) + return doc + + async with AsyncClient(app=app, base_url="http://test") as ac: + response = await ac.post("/embed/", data=input_doc.json()) + + doc = OutputDoc.parse_raw(response.content.decode()) + assert doc.embedding.shape == (100, 1) From 5c393bff9b7d316c8df8084faef3f106eae850fa Mon Sep 17 00:00:00 2001 From: samsja Date: Mon, 23 Jan 2023 14:59:19 +0100 Subject: [PATCH 4/9] chore: downgrade numpy Signed-off-by: samsja --- pyproject.toml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f41df6db23a..e0188310b2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,20 +6,19 @@ authors=['DocArray'] license='Apache 2.0' [tool.poetry.dependencies] -python = ">=3.7" -pydantic = ">=1.10.2" -numpy = ">=1.17.3" -protobuf = { version = ">=3.19.0", optional = true } -torch = { version = ">=1.0.0", optional = true } -orjson = ">=3.8.2" -pillow = {version = ">=9.3.0", optional = true } -types-pillow = {version = ">=9.3.0.1", optional = true } -trimesh = {version = ">=3.17.1", optional = true} -typing-inspect = ">=0.8.0" -types-requests = ">=2.28.11.6" -av = {version = ">=10.0.0", optional = true} -fastapi = {version = ">=0.87.0", optional = true } -rich = ">=13.1.0" +python = "^3.8" +pydantic = "^1.10.2" +numpy = "^1.22" +protobuf = { version = "^4.21.9", optional = true } +torch = { version = "^1.0.0", optional = true } +orjson = "^3.8.2" +pillow = {version = "^9.3.0", optional = true } +types-pillow = {version = "^9.3.0.1", optional = true } +trimesh = {version = "^3.17.1", optional = true} +typing-inspect = "^0.8.0" +types-requests = "^2.28.11.6" +av = {version = "^10.0.0", optional = true} +fastapi = {version = "^0.87.0", optional = true } [tool.poetry.extras] common = ["protobuf"] From 0a0e43b355861371cf7885d4577916de10174a0a Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 7 Feb 2023 10:33:30 +0100 Subject: [PATCH 5/9] fix: fix pypojrect oml Signed-off-by: samsja --- pyproject.toml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e0188310b2d..f41df6db23a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,19 +6,20 @@ authors=['DocArray'] license='Apache 2.0' [tool.poetry.dependencies] -python = "^3.8" -pydantic = "^1.10.2" -numpy = "^1.22" -protobuf = { version = "^4.21.9", optional = true } -torch = { version = "^1.0.0", optional = true } -orjson = "^3.8.2" -pillow = {version = "^9.3.0", optional = true } -types-pillow = {version = "^9.3.0.1", optional = true } -trimesh = {version = "^3.17.1", optional = true} -typing-inspect = "^0.8.0" -types-requests = "^2.28.11.6" -av = {version = "^10.0.0", optional = true} -fastapi = {version = "^0.87.0", optional = true } +python = ">=3.7" +pydantic = ">=1.10.2" +numpy = ">=1.17.3" +protobuf = { version = ">=3.19.0", optional = true } +torch = { version = ">=1.0.0", optional = true } +orjson = ">=3.8.2" +pillow = {version = ">=9.3.0", optional = true } +types-pillow = {version = ">=9.3.0.1", optional = true } +trimesh = {version = ">=3.17.1", optional = true} +typing-inspect = ">=0.8.0" +types-requests = ">=2.28.11.6" +av = {version = ">=10.0.0", optional = true} +fastapi = {version = ">=0.87.0", optional = true } +rich = ">=13.1.0" [tool.poetry.extras] common = ["protobuf"] From b5961da16bc742ddbd4bb9e040d45dcd761e345e Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 7 Feb 2023 10:35:00 +0100 Subject: [PATCH 6/9] fix: fix test Signed-off-by: samsja --- tests/integrations/externals/test_fastapi.py | 26 +------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/tests/integrations/externals/test_fastapi.py b/tests/integrations/externals/test_fastapi.py index c15e0632c91..f74f8519a9c 100644 --- a/tests/integrations/externals/test_fastapi.py +++ b/tests/integrations/externals/test_fastapi.py @@ -6,7 +6,7 @@ from docarray import BaseDocument from docarray.base_document import DocumentResponse from docarray.documents import Image, Text -from docarray.typing import AnyTensor, NdArray +from docarray.typing import NdArray @pytest.mark.asyncio @@ -107,27 +107,3 @@ async def create_item(doc: InputDoc) -> OutputDoc: assert isinstance(doc, OutputDoc) assert doc.embedding_clip.shape == (100, 1) assert doc.embedding_bert.shape == (100, 1) - - -@pytest.mark.asyncio -async def test_image(): - class InputDoc(BaseDocument): - img: Image - - class OutputDoc(BaseDocument): - embedding: AnyTensor - - input_doc = InputDoc(img=Image(tensor=np.zeros((3, 224, 224)))) - - app = FastAPI() - - @app.post("/embed/", response_model=OutputDoc, response_class=DocumentResponse) - async def embed(doc: InputDoc) -> OutputDoc: - doc = OutputDoc(embedding=np.zeros((100, 1))) - return doc - - async with AsyncClient(app=app, base_url="http://test") as ac: - response = await ac.post("/embed/", data=input_doc.json()) - - doc = OutputDoc.parse_raw(response.content.decode()) - assert doc.embedding.shape == (100, 1) From 035528ed9e46e46ca297147d7f674f1939865422 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 7 Feb 2023 10:45:18 +0100 Subject: [PATCH 7/9] fix: add docstring Signed-off-by: samsja --- docarray/documents/legacy/__init__.py | 5 +--- docarray/documents/legacy/legacy_document.py | 27 +++++++++++++++++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docarray/documents/legacy/__init__.py b/docarray/documents/legacy/__init__.py index 65ff29d224b..78e6b7bc1b1 100644 --- a/docarray/documents/legacy/__init__.py +++ b/docarray/documents/legacy/__init__.py @@ -1,6 +1,3 @@ -from docarray import DocumentArray as BaseDocumentArray from docarray.documents.legacy.legacy_document import Document -DocumentArray = BaseDocumentArray[Document] - -__all__ = ['Document', 'DocumentArray'] +__all__ = ['Document'] diff --git a/docarray/documents/legacy/legacy_document.py b/docarray/documents/legacy/legacy_document.py index 7f878c1edc3..4fe61fd5e57 100644 --- a/docarray/documents/legacy/legacy_document.py +++ b/docarray/documents/legacy/legacy_document.py @@ -7,6 +7,31 @@ class Document(BaseDocument): + """ + This Document is the LegacyDocument. It follows the same schema as in DocArray v1. + It can be useful to start migrating a codebase from v1 to v2. + + Nevertheless, the API is not totally compatible with DocAray v1 `Document`. + Indeed, none of the method associated with `Document` are present. Only the schema + of the data is similar. + .. code-block:: python + + from docarray import DocumentArray + from docarray.documents.legacy import Document + import numpy as np + + doc = Document(text='hello') + doc.url = 'http://myimg.png' + doc.tensor = np.zeros((3, 224, 224)) + doc.embedding = np.zeros((100, 1)) + + doc.tags['price'] = 10 + + doc.chunks = DocumentArray[Document]([Document() for _ in range(10)]) + + doc.chunks = DocumentArray[Document]([Document() for _ in range(10)]) + + """ tensor: Optional[AnyTensor] chunks: Optional[DocumentArray[Document]] @@ -15,5 +40,5 @@ class Document(BaseDocument): text: Optional[str] url: Optional[str] embedding: Optional[AnyEmbedding] - # tags: Dict[str, Any] = dict() + tags: Dict[str, Any] = dict() scores: Optional[Dict[str, Any]] From b38e6531753ddc7dad182efb8b6ccc76fb60fbb5 Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 7 Feb 2023 16:16:19 +0100 Subject: [PATCH 8/9] refactor: rename to legacy Signed-off-by: samsja --- docarray/documents/legacy/__init__.py | 4 ++-- docarray/documents/legacy/legacy_document.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docarray/documents/legacy/__init__.py b/docarray/documents/legacy/__init__.py index 78e6b7bc1b1..61cb9c485c1 100644 --- a/docarray/documents/legacy/__init__.py +++ b/docarray/documents/legacy/__init__.py @@ -1,3 +1,3 @@ -from docarray.documents.legacy.legacy_document import Document +from docarray.documents.legacy.legacy_document import LegacyDocument -__all__ = ['Document'] +__all__ = ['LegacyDocument'] diff --git a/docarray/documents/legacy/legacy_document.py b/docarray/documents/legacy/legacy_document.py index 4fe61fd5e57..08c5847dff2 100644 --- a/docarray/documents/legacy/legacy_document.py +++ b/docarray/documents/legacy/legacy_document.py @@ -6,7 +6,7 @@ from docarray.typing import AnyEmbedding, AnyTensor -class Document(BaseDocument): +class LegacyDocument(BaseDocument): """ This Document is the LegacyDocument. It follows the same schema as in DocArray v1. It can be useful to start migrating a codebase from v1 to v2. @@ -17,10 +17,10 @@ class Document(BaseDocument): .. code-block:: python from docarray import DocumentArray - from docarray.documents.legacy import Document + from docarray.documents.legacy import LegacyDocument import numpy as np - doc = Document(text='hello') + doc = LegacyDocument(text='hello') doc.url = 'http://myimg.png' doc.tensor = np.zeros((3, 224, 224)) doc.embedding = np.zeros((100, 1)) @@ -34,8 +34,8 @@ class Document(BaseDocument): """ tensor: Optional[AnyTensor] - chunks: Optional[DocumentArray[Document]] - matches: Optional[DocumentArray[Document]] + chunks: Optional[DocumentArray[LegacyDocument]] + matches: Optional[DocumentArray[LegacyDocument]] blob: Optional[Any] text: Optional[str] url: Optional[str] From 8320b1e3a0435c6c24d7cd367c87d1fc1174cb4a Mon Sep 17 00:00:00 2001 From: samsja Date: Tue, 7 Feb 2023 16:16:28 +0100 Subject: [PATCH 9/9] refactor: rename to legacy Signed-off-by: samsja --- docarray/documents/legacy/legacy_document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docarray/documents/legacy/legacy_document.py b/docarray/documents/legacy/legacy_document.py index 08c5847dff2..e3bd0c3b20c 100644 --- a/docarray/documents/legacy/legacy_document.py +++ b/docarray/documents/legacy/legacy_document.py @@ -36,7 +36,7 @@ class LegacyDocument(BaseDocument): tensor: Optional[AnyTensor] chunks: Optional[DocumentArray[LegacyDocument]] matches: Optional[DocumentArray[LegacyDocument]] - blob: Optional[Any] + blob: Optional[bytes] text: Optional[str] url: Optional[str] embedding: Optional[AnyEmbedding]