diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b88d16..088fd56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [0.0.8](https://github.com/cloudquery/plugin-sdk-python/compare/v0.0.7...v0.0.8) (2023-08-01) + + +### Bug Fixes + +* **deps:** Update dependency numpy to v1.25.2 ([#11](https://github.com/cloudquery/plugin-sdk-python/issues/11)) ([0d05fc8](https://github.com/cloudquery/plugin-sdk-python/commit/0d05fc8205c9d6f2c35b82647babef25dfd550c5)) +* Fix JSON and UUID type checking, add JSON test, consolidate setup.py ([#14](https://github.com/cloudquery/plugin-sdk-python/issues/14)) ([7927d1a](https://github.com/cloudquery/plugin-sdk-python/commit/7927d1aa4ca0f34252e7bfcccacc92d4a0975d46)) +* Remove resolver() from schema.Table ([#15](https://github.com/cloudquery/plugin-sdk-python/issues/15)) ([c61a774](https://github.com/cloudquery/plugin-sdk-python/commit/c61a7741a8a8c6d88e10cc91b15b41e5b83bccf0)) +* SyncMigrateTableMessage should have the `pa.Schema` argument named as "table" ([#13](https://github.com/cloudquery/plugin-sdk-python/issues/13)) ([a50f0e7](https://github.com/cloudquery/plugin-sdk-python/commit/a50f0e7a82b314a870f8195278ebe2bf9eb5442a)) + ## [0.0.7](https://github.com/cloudquery/plugin-sdk-python/compare/v0.0.6...v0.0.7) (2023-07-29) diff --git a/cloudquery/sdk/internal/servers/plugin_v3/plugin.py b/cloudquery/sdk/internal/servers/plugin_v3/plugin.py index 77f7df5..b3c7390 100644 --- a/cloudquery/sdk/internal/servers/plugin_v3/plugin.py +++ b/cloudquery/sdk/internal/servers/plugin_v3/plugin.py @@ -61,7 +61,7 @@ def Sync(self, request, context): insert=plugin_pb2.Sync.MessageInsert(record=buf) ) elif isinstance(msg, SyncMigrateTableMessage): - buf = arrow.schema_to_bytes(msg.schema) + buf = arrow.schema_to_bytes(msg.table) yield plugin_pb2.Sync.Response( migrate_table=plugin_pb2.Sync.MessageMigrateTable(table=buf) ) diff --git a/cloudquery/sdk/message/sync.py b/cloudquery/sdk/message/sync.py index d6741c0..c180edc 100644 --- a/cloudquery/sdk/message/sync.py +++ b/cloudquery/sdk/message/sync.py @@ -5,11 +5,11 @@ class SyncMessage: pass -class SyncInsertMessage: +class SyncInsertMessage(SyncMessage): def __init__(self, record: pa.RecordBatch): self.record = record -class SyncMigrateTableMessage: - def __init__(self, schema: pa.Schema): - self.schema = schema +class SyncMigrateTableMessage(SyncMessage): + def __init__(self, table: pa.Schema): + self.table = table diff --git a/cloudquery/sdk/scalar/scalar_factory.py b/cloudquery/sdk/scalar/scalar_factory.py index 4b390c1..6eabaca 100644 --- a/cloudquery/sdk/scalar/scalar_factory.py +++ b/cloudquery/sdk/scalar/scalar_factory.py @@ -1,5 +1,6 @@ import pyarrow as pa -from .scalar import ScalarInvalidTypeError + +from cloudquery.sdk.types import UUIDType, JSONType from .binary import Binary from .bool import Bool from .date32 import Date32 @@ -7,11 +8,11 @@ from .float import Float from .int import Int from .list import List +from .scalar import ScalarInvalidTypeError from .string import String from .timestamp import Timestamp from .uint import Uint from .uuid import UUID -from cloudquery.sdk.types import UUIDType, JSONType class ScalarFactory: @@ -85,9 +86,9 @@ def new_scalar(self, dt: pa.DataType): # return () elif dt_id == pa.types.lib.Type_TIMESTAMP: return Timestamp() - elif dt == UUIDType: + elif dt == UUIDType(): return UUID() - elif dt == JSONType: + elif dt == JSONType(): return String() else: raise ScalarInvalidTypeError("Invalid type {} for scalar".format(dt)) diff --git a/cloudquery/sdk/scheduler/scheduler.py b/cloudquery/sdk/scheduler/scheduler.py index fa94506..ae9b7ba 100644 --- a/cloudquery/sdk/scheduler/scheduler.py +++ b/cloudquery/sdk/scheduler/scheduler.py @@ -1,18 +1,16 @@ -from typing import List, Generator, Any import queue -import time +from concurrent import futures +from typing import List, Generator, Any + import structlog -from enum import Enum -from cloudquery.sdk.schema import Table, Resource + from cloudquery.sdk.message import ( SyncMessage, SyncInsertMessage, SyncMigrateTableMessage, ) -from concurrent import futures -from typing import Generator +from cloudquery.sdk.schema import Resource from .table_resolver import TableResolver -import traceback QUEUE_PER_WORKER = 100 @@ -174,7 +172,7 @@ def sync( ) -> Generator[SyncMessage, None, None]: res = queue.Queue() for resolver in resolvers: - yield SyncMigrateTableMessage(schema=resolver.table.to_arrow_schema()) + yield SyncMigrateTableMessage(table=resolver.table.to_arrow_schema()) thread = futures.ThreadPoolExecutor() thread.submit(self._sync, client, resolvers, res, deterministic_cq_id) total_table_resolvers = 0 @@ -192,4 +190,4 @@ def sync( break continue yield message - thread.shutdown() + thread.shutdown(wait=True) diff --git a/cloudquery/sdk/schema/table.py b/cloudquery/sdk/schema/table.py index 785614f..d0b56ef 100644 --- a/cloudquery/sdk/schema/table.py +++ b/cloudquery/sdk/schema/table.py @@ -36,9 +36,6 @@ def __init__( def multiplex(self, client) -> List[Table]: raise [client] - def resolver(self, client: Client, parent=None) -> Generator[Any]: - raise NotImplementedError() - def index_column(self, column_name: str) -> int: for i, column in enumerate(self.columns): if column.name == column_name: diff --git a/requirements.txt b/requirements.txt index b4871e2..945c9b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1 @@ -cloudquery-plugin-pb==0.0.14 -exceptiongroup==1.1.2 -black==23.7.0 -grpcio==1.56.2 -grpcio-tools==1.56.2 -iniconfig==2.0.0 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -numpy==1.25.1 -packaging==23.1 -pandas==2.0.3 -pluggy==1.2.0 -protobuf==4.23.4 -pyarrow==12.0.1 -pytest==7.4.0 -python-dateutil==2.8.2 -pytz==2023.3 -six==1.16.0 -structlog==23.1.0 -tomli==2.0.1 -tzdata==2023.3 +. \ No newline at end of file diff --git a/setup.py b/setup.py index 6f1847c..40c514a 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,26 @@ dependencies = [ "cloudquery-plugin-pb==0.0.14", - "pyarrow==12.0.1", + "exceptiongroup==1.1.2", + "black==23.7.0", + "grpcio==1.56.2", + "grpcio-tools==1.56.2", + "iniconfig==2.0.0", "Jinja2==3.1.2", + "MarkupSafe==2.1.3", + "numpy==1.25.2", + "packaging==23.1", + "pandas==2.0.3", + "pluggy==1.2.0", + "protobuf==4.23.4", + "pyarrow==12.0.1", + "pytest==7.4.0", + "python-dateutil==2.8.2", + "pytz==2023.3", + "six==1.16.0", "structlog==23.1.0", + "tomli==2.0.1", + "tzdata==2023.3", ] url = "https://github.com/cloudquery/plugin-sdk-python" @@ -36,7 +53,7 @@ ] setuptools.setup( name=name, - version="0.0.7", + version="0.0.8", description=description, long_description=long_description, author="CloudQuery LTD", diff --git a/tests/types/__init__.py b/tests/types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/types/json.py b/tests/types/json.py new file mode 100644 index 0000000..f0ba704 --- /dev/null +++ b/tests/types/json.py @@ -0,0 +1,7 @@ +from cloudquery.sdk.types import JSONType + + +def test_json_type(): + j = JSONType() + # test equality + assert j == JSONType()